patchwork

package doc

import (
	"unicode"
	"unicode/utf8"
)

// Tokenize splits text into words, whitespace runs and single punctuation
// marks. Concatenating the tokens gives back the input exactly. An apostrophe
// between letters stays inside the word ("don't", "d'água").
func Tokenize(s string) []string {
	var out []string
	for i := 0; i < len(s); {
		r, size := utf8.DecodeRuneInString(s[i:])
		j := i + size
		switch {
		case isWordRune(r):
			for j < len(s) {
				r2, s2 := utf8.DecodeRuneInString(s[j:])
				if isWordRune(r2) {
					j += s2
					continue
				}
				if r2 == '\'' || r2 == '’' {
					if r3, _ := utf8.DecodeRuneInString(s[j+s2:]); j+s2 < len(s) && unicode.IsLetter(r3) {
						j += s2
						continue
					}
				}
				break
			}
		case unicode.IsSpace(r):
			for j < len(s) {
				r2, s2 := utf8.DecodeRuneInString(s[j:])
				if !unicode.IsSpace(r2) {
					break
				}
				j += s2
			}
		}
		out = append(out, s[i:j])
		i = j
	}
	return out
}

// IsWord reports whether a token from Tokenize is a word.
func IsWord(tok string) bool {
	r, _ := utf8.DecodeRuneInString(tok)
	return isWordRune(r)
}

func isWordRune(r rune) bool {
	return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) || r == '_'
}