chore(deps): resolve dependency conflict

This commit is contained in:
Chris Lane
2020-05-11 20:12:28 -04:00
parent 1b17ab1914
commit 4eeec6c868
92 changed files with 1750 additions and 317 deletions

View File

@ -410,6 +410,9 @@ func (r *RegexLexer) Tokenise(options *TokeniseOptions, text string) (Iterator,
if options == nil {
options = defaultOptions
}
if options.EnsureLF {
text = ensureLF(text)
}
if !options.Nested && r.config.EnsureNL && !strings.HasSuffix(text, "\n") {
text += "\n"
}
@ -437,3 +440,22 @@ func matchRules(text []rune, pos int, rules []*CompiledRule) (int, *CompiledRule
}
return 0, &CompiledRule{}, nil
}
// replace \r and \r\n with \n
// same as strings.ReplaceAll but more efficient
func ensureLF(text string) string {
buf := make([]byte, len(text))
var j int
for i := 0; i < len(text); i++ {
c := text[i]
if c == '\r' {
if i < len(text)-1 && text[i+1] == '\n' {
continue
}
c = '\n'
}
buf[j] = c
j++
}
return string(buf[:j])
}