// splitSentences splits the string into sentences using the rules for creating
// koboSpans. To make this zero-allocation, pass a zero-length slice for
// splitSentences to take ownership of. To re-use the slice, pass the returned
// slice, sliced to zero. If the slice is too small, it will be grown, causing
// an allocation.
//
// This state-machine based implementation is around three times as fast as the
// regexp-based one on average, and even faster when pre-allocating and re-using
// the sentences slice. It should have the same output. For the original
// implementation, see splitSentencesRegexp in the tests.
splitSentences splits the string into sentences using the rules for creating
koboSpans. To make this zero-allocation, pass a zero-length slice for
splitSentences to take ownership of. To re-use the slice, pass the returned
slice, sliced to zero. If the slice is too small, it will be grown, causing
an allocation.

This state-machine based implementation is around three times as fast as the
regexp-based one on average, and even faster when pre-allocating and re-using
the sentences slice. It should have the same output. For the original
implementation, see splitSentencesRegexp in the tests.
// splitSentences splits the string into sentences using the rules for creating // koboSpans . To make this zero - allocation pass a zero - length slice for // splitSentences to take ownership of . To re - use the slice pass the returned // slice sliced to zero . If the slice is too small it will be grown causing // an allocation . // // This state - machine based implementation is around three times as fast as the // regexp - based one on average and even faster when pre - allocating and re - using // the sentences slice . It should have the same output . For the original // implementation see splitSentencesRegexp in the tests .
func splitSentences(str string, sentences []string) []string {
	const (
		InputPunct   = iota 
		InputExtra          
		InputSpace          
		InputAny            
		InputInvalid        
		InputEOS            
	)
	const (
		OutputNone = iota 
		OutputNext        
		OutputRest        
	)
	const (
		StateDefault         = iota 
		StateAfterPunct             
		StateAfterPunctExtra        
		StateAfterSpace             
	)
	if sentences == nil {
		sentences = make([]string, 0, 4) 
	}
	for i, state := 0, 0; state != -1; {
		x, z := utf8.DecodeRuneInString(str[i:])
		var input int
		switch x {
		case utf8.RuneError:
			switch z {
			case 0:
				input = InputEOS
			default:
				input = InputInvalid
			}
		case '.', '!', '?':
			input = InputPunct
		case '\'', '"', '”', '’', '“', '…':
			input = InputExtra
		case '\t', '\n', '\f', '\r', ' ': 
			input = InputSpace
		default:
			input = InputAny
		}
		var output int
		switch state {
		case StateDefault:
			switch input {
			case InputPunct:
				output, state = OutputNone, StateAfterPunct
			case InputExtra:
				output, state = OutputNone, StateDefault
			case InputSpace:
				output, state = OutputNone, StateDefault
			case InputAny:
				output, state = OutputNone, StateDefault
			case InputInvalid:
				output, state = OutputNone, StateDefault
			case InputEOS:
				output, state = OutputRest, -1
			default:
				panic("unhandled input")
			}
		case StateAfterPunct:
			switch input {
			case InputPunct:
				output, state = OutputNone, StateAfterPunct
			case InputExtra:
				output, state = OutputNone, StateAfterPunctExtra
			case InputSpace:
				output, state = OutputNone, StateAfterSpace
			case InputAny:
				output, state = OutputNone, StateDefault
			case InputInvalid:
				output, state = OutputNone, StateDefault
			case InputEOS:
				output, state = OutputRest, -1
			default:
				panic("unhandled input")
			}
		case StateAfterPunctExtra:
			switch input {
			case InputPunct:
				output, state = OutputNone, StateAfterPunct
			case InputExtra:
				output, state = OutputNone, StateDefault
			case InputSpace:
				output, state = OutputNone, StateAfterSpace
			case InputAny:
				output, state = OutputNone, StateDefault
			case InputInvalid:
				output, state = OutputNone, StateDefault
			case InputEOS:
				output, state = OutputRest, -1
			default:
				panic("unhandled input")
			}
		case StateAfterSpace:
			switch input {
			case InputPunct:
				output, state = OutputNext, StateAfterPunct
			case InputExtra:
				output, state = OutputNext, StateDefault
			case InputSpace:
				output, state = OutputNone, StateAfterSpace
			case InputAny:
				output, state = OutputNext, StateDefault
			case InputInvalid:
				output, state = OutputNext, StateDefault
			case InputEOS:
				output, state = OutputRest, -1
			default:
				panic("unhandled input")
			}
		default:
			panic("unhandled state")
		}
		switch output {
		case OutputNone:
			i += z
		case OutputNext:
			sentences = append(sentences, str[:i])
			str, i = str[i:], z
		case OutputRest:
			if len(str) != 0 || len(sentences) == 0 {
				sentences = append(sentences, str)
			}
			if state != -1 {
				panic("invalid state")
			}
		default:
			panic("unhandled output")
		}
	}
	return sentences
}
func splitSentences ( str string , sentences [ ] string ) [ ] string { const ( InputPunct = iota InputExtra InputSpace InputAny InputInvalid InputEOS ) const ( OutputNone = iota OutputNext OutputRest ) const ( StateDefault = iota StateAfterPunct StateAfterPunctExtra StateAfterSpace ) if sentences == nil { sentences = make ( [ ] string , 0 , 4 ) } for i , state := 0 , 0 ; state != - 1 ; { x , z := utf8 . DecodeRuneInString ( str [ i : ] ) var input int switch x { case utf8 . RuneError : switch z { case 0 : input = InputEOS default : input = InputInvalid } case '.' , '!' , '?' : input = InputPunct case '\'' , '"' , '”',  ' ', '“ '  '…': input = InputExtra case '\t' , '\n' , '\f' , '\r' , ' ' : input = InputSpace default : input = InputAny } var output int switch state { case StateDefault : switch input { case InputPunct : output , state = OutputNone , StateAfterPunct case InputExtra : output , state = OutputNone , StateDefault case InputSpace : output , state = OutputNone , StateDefault case InputAny : output , state = OutputNone , StateDefault case InputInvalid : output , state = OutputNone , StateDefault case InputEOS : output , state = OutputRest , - 1 default : panic ( " " ) } case StateAfterPunct : switch input { case InputPunct : output , state = OutputNone , StateAfterPunct case InputExtra : output , state = OutputNone , StateAfterPunctExtra case InputSpace : output , state = OutputNone , StateAfterSpace case InputAny : output , state = OutputNone , StateDefault case InputInvalid : output , state = OutputNone , StateDefault case InputEOS : output , state = OutputRest , - 1 default : panic ( " " ) } case StateAfterPunctExtra : switch input { case InputPunct : output , state = OutputNone , StateAfterPunct case InputExtra : output , state = OutputNone , StateDefault case InputSpace : output , state = OutputNone , StateAfterSpace case InputAny : output , state = OutputNone , StateDefault case InputInvalid : output , state = OutputNone , StateDefault case InputEOS : output , state = OutputRest , - 1 default : panic ( " " ) } case StateAfterSpace : switch input { case InputPunct : output , state = OutputNext , StateAfterPunct case InputExtra : output , state = OutputNext , StateDefault case InputSpace : output , state = OutputNone , StateAfterSpace case InputAny : output , state = OutputNext , StateDefault case InputInvalid : output , state = OutputNext , StateDefault case InputEOS : output , state = OutputRest , - 1 default : panic ( " " ) } default : panic ( " " ) } switch output { case OutputNone : i += z case OutputNext : sentences = append ( sentences , str [ : i ] ) str , i = str [ i : ] , z case OutputRest : if len ( str ) != 0 || len ( sentences ) == 0 { sentences = append ( sentences , str ) } if state != - 1 { panic ( " " ) } default : panic ( " " ) } } return sentences }
splitSentences splits the string into sentences using the rules for creating
koboSpans.
splitSentences splits the string into sentences using the rules for creating koboSpans .
// sentence-terminating punctuation // additional punctuation (one is optionionally consumed after punct if present) // whitespace // any valid rune not previously matched // an invalid byte // end-of-string // moves to the next rune. // adds everything from the last call up to (but not including) the current rune, and moves to the next rune. // adds everything not yet added by OutputNext (state must be -1) // in a sentence // after the sentence-terminating rune // after the optional additional punctuation rune // the trailing whitespace after the sentence // pre-allocate some room // \s only matches only ASCII whitespace