// ScanWords is a split function for a Scanner that returns each
// space/punctuation-separated word of text, with surrounding spaces
// deleted. It will never return an empty string. The definition of
// space/punctuation is set by unicode.IsSpace and unicode.IsPunct.
ScanWords is a split function for a Scanner that returns each
space/punctuation-separated word of text, with surrounding spaces
deleted. It will never return an empty string. The definition of
space/punctuation is set by unicode.IsSpace and unicode.IsPunct.
// ScanWords is a split function for a Scanner that returns each // space / punctuation - separated word of text with surrounding spaces // deleted . It will never return an empty string . The definition of // space / punctuation is set by unicode . IsSpace and unicode . IsPunct .
func (w *words) ScanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	start := 0
	w.current.pos = w.current.end
	var prev rune
	for width, i := 0, 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !isSplitter(prev, r, data[i+width:]) {
			prev = r
			break
		}
		prev = r
	}
	w.current.pos += start
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isSplitter(prev, r, data[i+width:]) {
			w.current.end += i + width
			return i + width, data[start:i], nil
		}
		prev = r
	}
	if atEOF && len(data) > start {
		w.current.end += len(data)
		return len(data), data[start:], nil
	}
	w.current.end = w.current.pos
	return start, nil, nil
}
func ( w * words ) ScanWords ( data [ ] byte , atEOF bool ) ( advance int , token [ ] byte , err error ) { start := 0 w . current . pos = w . current . end var prev rune for width , i := 0 , 0 ; start < len ( data ) ; start += width { var r rune r , width = utf8 . DecodeRune ( data [ start : ] ) if ! isSplitter ( prev , r , data [ i + width : ] ) { prev = r break } prev = r } w . current . pos += start for width , i := 0 , start ; i < len ( data ) ; i += width { var r rune r , width = utf8 . DecodeRune ( data [ i : ] ) if isSplitter ( prev , r , data [ i + width : ] ) { w . current . end += i + width return i + width , data [ start : i ] , nil } prev = r } if atEOF && len ( data ) > start { w . current . end += len ( data ) return len ( data ) , data [ start : ] , nil } w . current . end = w . current . pos return start , nil , nil }
ScanWords is a split function for a Scanner that returns each
space/punctuation-separated word of text, with surrounding spaces
deleted.
ScanWords is a split function for a Scanner that returns each space / punctuation - separated word of text with surrounding spaces deleted .
// Scan until space, marking end of word. // If we're at EOF, we have a final, non-empty, non-terminated word. Return it. // Request more data.