public List<NamedEntityTask1Data> generateTask1Data(List<JCas> documentsJCas,int goldOffset,boolean generateGold,int limit) throws CrowdException {
  Log LOG=LogFactory.getLog(getClass());
  List<NamedEntityTask1Data> data=new ArrayList<NamedEntityTask1Data>();
  int i=goldOffset;
  StringBuilder textBuilder=new StringBuilder();
  int docNo=0;
  jcasloop:   for (  JCas documentJCas : documentsJCas) {
    int offset=i;
    int sentenceNo=0;
    LOG.info("Generating data for document: " + docNo + "/"+ documentsJCas.size());
    for (    Sentence sentence : select(documentJCas,Sentence.class)) {
      if (limit != -1 && sentenceNo >= limit) {
        break jcasloop;
      }
      textBuilder.setLength(0);
      Map<Integer,Integer> charOffsetStartMapping=new HashMap<Integer,Integer>();
      Map<Integer,Integer> charOffsetEndMapping=new HashMap<Integer,Integer>();
      for (      Token token : selectCovered(Token.class,sentence)) {
        textBuilder.append(HTML_SPAN_TOKEN_START);
        textBuilder.append(String.valueOf(i));
        textBuilder.append(HTML_SPAN_TOKEN_CLOSE);
        textBuilder.append(escapeHtml(token.getCoveredText()));
        textBuilder.append(" ");
        textBuilder.append(HTML_SPAN_CLOSE);
        charOffsetStartMapping.put(token.getBegin(),i);
        charOffsetEndMapping.put(token.getEnd(),i);
        i++;
      }
      String text=textBuilder.toString();
      textBuilder.setLength(0);
      NamedEntityTask1Data task1Data=new NamedEntityTask1Data(text);
      task1Data.setOffset(offset);
      task1Data.setDocument(JSON_SOURCEDOC_SHORTFORM + String.valueOf(docNo));
      if (generateGold) {
        List<String> goldElems=new ArrayList<String>();
        List<String> goldTokens=new ArrayList<String>();
        List<String> goldTypes=new ArrayList<String>();
        int lastNeBegin=-1;
        int lastNeEnd=-1;
        for (        NamedEntity namedEntity : selectCovered(NamedEntity.class,sentence)) {
          List<Token> tokens=selectCovered(documentJCas,Token.class,namedEntity.getBegin(),namedEntity.getEnd());
          List<String> strTokens=new ArrayList<String>();
          for (          Token t : tokens) {
            strTokens.add(new String(t.getCoveredText()));
          }
          String strToken=concatWithSeparator(strTokens," ");
          String type=namedEntity.getValue();
          int neBegin=namedEntity.getBegin();
          int neEnd=namedEntity.getEnd();
          String strElem=buildTask1TokenJSON(textBuilder,namedEntity,charOffsetStartMapping,charOffsetEndMapping);
          if (lastNeEnd != -1 && neBegin <= lastNeEnd) {
            if ((neEnd - neBegin) > (lastNeEnd - lastNeBegin)) {
              goldElems.remove(goldElems.size() - 1);
              goldTokens.remove(goldElems.size() - 1);
              goldTypes.remove(goldElems.size() - 1);
              goldElems.add(strElem);
              goldTokens.add(strToken);
              goldTypes.add(type);
              lastNeBegin=neBegin;
              lastNeEnd=neEnd;
            }
          }
 else {
            goldElems.add(strElem);
            goldTokens.add(strToken);
            goldTypes.add(type);
            lastNeBegin=neBegin;
            lastNeEnd=neEnd;
          }
        }
        String strTypes=JSON_VALUE_EMPTY_ARRAY;
        String strGold=JSON_VALUE_NONE_MARKER;
        String strGoldReason=noGoldNER1Reason;
        int difficulty=1;
        if (goldElems.size() > 0) {
          strGold=buildTask1GoldElem(textBuilder,goldElems);
          difficulty=goldElems.size();
          strTypes=buildTask1GoldElem(textBuilder,goldTypes);
          strGoldReason=buildTask1GoldElemReason(textBuilder,goldTokens);
        }
        task1Data.set_difficulty(difficulty);
        task1Data.setMarkertext_gold(strGold);
        task1Data.setMarkertext_gold_reason(strGoldReason);
        task1Data.setTypes(strTypes);
        task1Data.setDocument(JSON_GOLDDOC_SHORTFORM + String.valueOf(docNo));
        task1Data.set_golden(JSON_TRUE);
      }
      data.add(task1Data);
      sentenceNo++;
    }
    docNo++;
  }
  if (generateGold) {
    lastGoldOffset=i;
  }
  return data;
}

public List < NamedEntityTask1Data > generateTask1Data ( List < JCas > documentsJCas , int goldOffset , boolean generateGold , int limit ) throws CrowdException { Log LOG = LogFactory . getLog ( getClass ( ) ) ; List < NamedEntityTask1Data > data = new ArrayList < NamedEntityTask1Data > ( ) ; int i = goldOffset ; StringBuilder textBuilder = new StringBuilder ( ) ; int docNo = 0 ; jcasloop : for ( JCas documentJCas : documentsJCas ) { int offset = i ; int sentenceNo = 0 ; LOG . info ( "Generating data for document: " + docNo + "/" + documentsJCas . size ( ) ) ; for ( Sentence sentence : select ( documentJCas , Sentence . class ) ) { if ( limit != - 1 && sentenceNo >= limit ) { break jcasloop ; } textBuilder . setLength ( 0 ) ; Map < Integer , Integer > charOffsetStartMapping = new HashMap < Integer , Integer > ( ) ; Map < Integer , Integer > charOffsetEndMapping = new HashMap < Integer , Integer > ( ) ; for ( Token token : selectCovered ( Token . class , sentence ) ) { textBuilder . append ( HTML_SPAN_TOKEN_START ) ; textBuilder . append ( String . valueOf ( i ) ) ; textBuilder . append ( HTML_SPAN_TOKEN_CLOSE ) ; textBuilder . append ( escapeHtml ( token . getCoveredText ( ) ) ) ; textBuilder . append ( " " ) ; textBuilder . append ( HTML_SPAN_CLOSE ) ; charOffsetStartMapping . put ( token . getBegin ( ) , i ) ; charOffsetEndMapping . put ( token . getEnd ( ) , i ) ; i ++ ; } String text = textBuilder . toString ( ) ; textBuilder . setLength ( 0 ) ; NamedEntityTask1Data task1Data = new NamedEntityTask1Data ( text ) ; task1Data . setOffset ( offset ) ; task1Data . setDocument ( JSON_SOURCEDOC_SHORTFORM + String . valueOf ( docNo ) ) ; if ( generateGold ) { List < String > goldElems = new ArrayList < String > ( ) ; List < String > goldTokens = new ArrayList < String > ( ) ; List < String > goldTypes = new ArrayList < String > ( ) ; int lastNeBegin = - 1 ; int lastNeEnd = - 1 ; for ( NamedEntity namedEntity : selectCovered ( NamedEntity . class , sentence ) ) { List < Token > tokens = selectCovered ( documentJCas , Token . class , namedEntity . getBegin ( ) , namedEntity . getEnd ( ) ) ; List < String > strTokens = new ArrayList < String > ( ) ; for ( Token t : tokens ) { strTokens . add ( new String ( t . getCoveredText ( ) ) ) ; } String strToken = concatWithSeparator ( strTokens , " " ) ; String type = namedEntity . getValue ( ) ; int neBegin = namedEntity . getBegin ( ) ; int neEnd = namedEntity . getEnd ( ) ; String strElem = buildTask1TokenJSON ( textBuilder , namedEntity , charOffsetStartMapping , charOffsetEndMapping ) ; if ( lastNeEnd != - 1 && neBegin <= lastNeEnd ) { if ( ( neEnd - neBegin ) > ( lastNeEnd - lastNeBegin ) ) { goldElems . remove ( goldElems . size ( ) - 1 ) ; goldTokens . remove ( goldElems . size ( ) - 1 ) ; goldTypes . remove ( goldElems . size ( ) - 1 ) ; goldElems . add ( strElem ) ; goldTokens . add ( strToken ) ; goldTypes . add ( type ) ; lastNeBegin = neBegin ; lastNeEnd = neEnd ; } } else { goldElems . add ( strElem ) ; goldTokens . add ( strToken ) ; goldTypes . add ( type ) ; lastNeBegin = neBegin ; lastNeEnd = neEnd ; } } String strTypes = JSON_VALUE_EMPTY_ARRAY ; String strGold = JSON_VALUE_NONE_MARKER ; String strGoldReason = noGoldNER1Reason ; int difficulty = 1 ; if ( goldElems . size ( ) > 0 ) { strGold = buildTask1GoldElem ( textBuilder , goldElems ) ; difficulty = goldElems . size ( ) ; strTypes = buildTask1GoldElem ( textBuilder , goldTypes ) ; strGoldReason = buildTask1GoldElemReason ( textBuilder , goldTokens ) ; } task1Data . set_difficulty ( difficulty ) ; task1Data . setMarkertext_gold ( strGold ) ; task1Data . setMarkertext_gold_reason ( strGoldReason ) ; task1Data . setTypes ( strTypes ) ; task1Data . setDocument ( JSON_GOLDDOC_SHORTFORM + String . valueOf ( docNo ) ) ; task1Data . set_golden ( JSON_TRUE ) ; } data . add ( task1Data ) ; sentenceNo ++ ; } docNo ++ ; } if ( generateGold ) { lastGoldOffset = i ; } return data ; }
generate data for ner task1 . this is a quite specific function exclusively for german ner , but a more general version could one day replace this . you usually call this function twice , one time to generate gold data ( generategold = true ) and one time to generate normal data
