fix: resolves #498

Resolves an issue whereby background colors were written to the terminal
output when colorization was applied.
This commit is contained in:
Chris Lane 2019-11-07 20:24:08 -05:00
parent e08a4f3cec
commit 50dc3c8b29
28 changed files with 492 additions and 33 deletions

View File

@ -13,7 +13,7 @@ import (
"github.com/cheat/cheat/internal/config" "github.com/cheat/cheat/internal/config"
) )
const version = "3.0.5" const version = "3.0.6"
func main() { func main() {

2
go.mod
View File

@ -3,7 +3,7 @@ module github.com/cheat/cheat
go 1.13 go 1.13
require ( require (
github.com/alecthomas/chroma v0.6.7 github.com/alecthomas/chroma v0.6.8
github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew v1.1.1
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
github.com/mattn/go-isatty v0.0.10 github.com/mattn/go-isatty v0.0.10

2
go.sum
View File

@ -5,6 +5,8 @@ github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
github.com/alecthomas/chroma v0.6.7 h1:1hKci+AyKOxJrugR9veaocu9DQGR2/GecI72BpaO0Rg= github.com/alecthomas/chroma v0.6.7 h1:1hKci+AyKOxJrugR9veaocu9DQGR2/GecI72BpaO0Rg=
github.com/alecthomas/chroma v0.6.7/go.mod h1:zVlgtbRS7BJDrDY9SB238RmpoCBCYFlLmcfZ3durxTk= github.com/alecthomas/chroma v0.6.7/go.mod h1:zVlgtbRS7BJDrDY9SB238RmpoCBCYFlLmcfZ3durxTk=
github.com/alecthomas/chroma v0.6.8 h1:TW4JJaIdbAbMyUtGEd6BukFlOKYvVQz3vVhLBEUNwMU=
github.com/alecthomas/chroma v0.6.8/go.mod h1:o9ohftueRi7H5be3+Q2cQCNa/YnLBFUNx40ZJfGVFKA=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI=

View File

@ -15,6 +15,9 @@ linters:
- gocyclo - gocyclo
- dupl - dupl
- gochecknoglobals - gochecknoglobals
- funlen
- godox
- wsl
linters-settings: linters-settings:
govet: govet:
@ -43,3 +46,6 @@ issues:
- 'Potential file inclusion via variable' - 'Potential file inclusion via variable'
- 'should have comment or be unexported' - 'should have comment or be unexported'
- 'comment on exported var .* should be of the form' - 'comment on exported var .* should be of the form'
- 'at least one file in a package should have a package comment'
- 'string literal contains the Unicode'
- 'methods on the same type should have the same receiver name'

View File

@ -2,7 +2,7 @@ sudo: false
language: go language: go
script: script:
- go test -v ./... - go test -v ./...
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s v1.15.0 - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s v1.20.0
- ./bin/golangci-lint run - ./bin/golangci-lint run
- git clean -fdx . - git clean -fdx .
after_success: after_success:

View File

@ -44,7 +44,7 @@ F | Factor, Fish, Forth, Fortran, FSharp
G | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groovy G | GAS, GDScript, Genshi, Genshi HTML, Genshi Text, GLSL, Gnuplot, Go, Go HTML Template, Go Text Template, GraphQL, Groovy
H | Handlebars, Haskell, Haxe, HCL, Hexdump, HTML, HTTP, Hy H | Handlebars, Haskell, Haxe, HCL, Hexdump, HTML, HTTP, Hy
I | Idris, INI, Io I | Idris, INI, Io
J | Java, JavaScript, JSON, Julia, Jungle J | J, Java, JavaScript, JSON, Julia, Jungle
K | Kotlin K | Kotlin
L | Lighttpd configuration file, LLVM, Lua L | Lighttpd configuration file, LLVM, Lua
M | Mako, markdown, Mason, Mathematica, Matlab, MiniZinc, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL M | Mako, markdown, Mason, Mathematica, Matlab, MiniZinc, Modula-2, MonkeyC, MorrowindScript, Myghty, MySQL

View File

@ -34,7 +34,7 @@ type insertion struct {
tokens []Token tokens []Token
} }
func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { func (d *delegatingLexer) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { // nolint: gocognit
tokens, err := Tokenise(Coalesce(d.language), options, text) tokens, err := Tokenise(Coalesce(d.language), options, text)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -6,6 +6,7 @@ import (
"github.com/alecthomas/chroma" "github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters/html" "github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/formatters/svg"
) )
var ( var (
@ -20,6 +21,7 @@ var (
})) }))
// Default HTML formatter outputs self-contained HTML. // Default HTML formatter outputs self-contained HTML.
htmlFull = Register("html", html.New(html.Standalone(), html.WithClasses())) // nolint htmlFull = Register("html", html.New(html.Standalone(), html.WithClasses())) // nolint
SVG = Register("svg", svg.New(svg.EmbedFont("Liberation Mono", svg.FontLiberationMono, svg.WOFF)))
) )
// Fallback formatter. // Fallback formatter.

View File

@ -91,11 +91,6 @@ func (h highlightRanges) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
func (h highlightRanges) Less(i, j int) bool { return h[i][0] < h[j][0] } func (h highlightRanges) Less(i, j int) bool { return h[i][0] < h[j][0] }
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) { func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
defer func() {
if perr := recover(); perr != nil {
err = perr.(error)
}
}()
return f.writeHTML(w, style, iterator.Tokens()) return f.writeHTML(w, style, iterator.Tokens())
} }

File diff suppressed because one or more lines are too long

View File

@ -0,0 +1,222 @@
// Package svg contains an SVG formatter.
package svg
import (
"encoding/base64"
"errors"
"fmt"
"io"
"io/ioutil"
"path"
"strings"
"github.com/alecthomas/chroma"
)
// Option sets an option of the SVG formatter.
type Option func(f *Formatter)
// FontFamily sets the font-family.
func FontFamily(fontFamily string) Option { return func(f *Formatter) { f.fontFamily = fontFamily } }
// EmbedFontFile embeds given font file
func EmbedFontFile(fontFamily string, fileName string) (option Option, err error) {
var format FontFormat
switch path.Ext(fileName) {
case ".woff":
format = WOFF
case ".woff2":
format = WOFF2
case ".ttf":
format = TRUETYPE
default:
return nil, errors.New("unexpected font file suffix")
}
var content []byte
if content, err = ioutil.ReadFile(fileName); err == nil {
option = EmbedFont(fontFamily, base64.StdEncoding.EncodeToString(content), format)
}
return
}
// EmbedFont embeds given base64 encoded font
func EmbedFont(fontFamily string, font string, format FontFormat) Option {
return func(f *Formatter) { f.fontFamily = fontFamily; f.embeddedFont = font; f.fontFormat = format }
}
// New SVG formatter.
func New(options ...Option) *Formatter {
f := &Formatter{fontFamily: "Consolas, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, Bitstream Vera Sans Mono, Courier New, monospace"}
for _, option := range options {
option(f)
}
return f
}
// Formatter that generates SVG.
type Formatter struct {
fontFamily string
embeddedFont string
fontFormat FontFormat
}
func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
f.writeSVG(w, style, iterator.Tokens())
return err
}
var svgEscaper = strings.NewReplacer(
`&`, "&amp;",
`<`, "&lt;",
`>`, "&gt;",
`"`, "&quot;",
` `, "&#160;",
` `, "&#160;&#160;&#160;&#160;",
)
// EscapeString escapes special characters.
func escapeString(s string) string {
return svgEscaper.Replace(s)
}
func (f *Formatter) writeSVG(w io.Writer, style *chroma.Style, tokens []chroma.Token) { // nolint: gocyclo
svgStyles := f.styleToSVG(style)
lines := chroma.SplitTokensIntoLines(tokens)
fmt.Fprint(w, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
fmt.Fprint(w, "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.0//EN\" \"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd\">\n")
fmt.Fprintf(w, "<svg width=\"%dpx\" height=\"%dpx\" xmlns=\"http://www.w3.org/2000/svg\">\n", 8*maxLineWidth(lines), 10+int(16.8*float64(len(lines)+1)))
if f.embeddedFont != "" {
f.writeFontStyle(w)
}
fmt.Fprintf(w, "<rect width=\"100%%\" height=\"100%%\" fill=\"%s\"/>\n", style.Get(chroma.Background).Background.String())
fmt.Fprintf(w, "<g font-family=\"%s\" font-size=\"14px\" fill=\"%s\">\n", f.fontFamily, style.Get(chroma.Text).Colour.String())
f.writeTokenBackgrounds(w, lines, style)
for index, tokens := range lines {
fmt.Fprintf(w, "<text x=\"0\" y=\"%fem\" xml:space=\"preserve\">", 1.2*float64(index+1))
for _, token := range tokens {
text := escapeString(token.String())
attr := f.styleAttr(svgStyles, token.Type)
if attr != "" {
text = fmt.Sprintf("<tspan %s>%s</tspan>", attr, text)
}
fmt.Fprint(w, text)
}
fmt.Fprint(w, "</text>")
}
fmt.Fprint(w, "\n</g>\n")
fmt.Fprint(w, "</svg>\n")
}
func maxLineWidth(lines [][]chroma.Token) int {
maxWidth := 0
for _, tokens := range lines {
length := 0
for _, token := range tokens {
length += len(strings.Replace(token.String(), ` `, " ", -1))
}
if length > maxWidth {
maxWidth = length
}
}
return maxWidth
}
// There is no background attribute for text in SVG so simply calculate the position and text
// of tokens with a background color that differs from the default and add a rectangle for each before
// adding the token.
func (f *Formatter) writeTokenBackgrounds(w io.Writer, lines [][]chroma.Token, style *chroma.Style) {
for index, tokens := range lines {
lineLength := 0
for _, token := range tokens {
length := len(strings.Replace(token.String(), ` `, " ", -1))
tokenBackground := style.Get(token.Type).Background
if tokenBackground.IsSet() && tokenBackground != style.Get(chroma.Background).Background {
fmt.Fprintf(w, "<rect id=\"%s\" x=\"%dch\" y=\"%fem\" width=\"%dch\" height=\"1.2em\" fill=\"%s\" />\n", escapeString(token.String()), lineLength, 1.2*float64(index)+0.25, length, style.Get(token.Type).Background.String())
}
lineLength += length
}
}
}
type FontFormat int
// https://transfonter.org/formats
const (
WOFF FontFormat = iota
WOFF2
TRUETYPE
)
var fontFormats = [...]string{
"woff",
"woff2",
"truetype",
}
func (f *Formatter) writeFontStyle(w io.Writer) {
fmt.Fprintf(w, `<style>
@font-face {
font-family: '%s';
src: url(data:application/x-font-%s;charset=utf-8;base64,%s) format('%s');'
font-weight: normal;
font-style: normal;
}
</style>`, f.fontFamily, fontFormats[f.fontFormat], f.embeddedFont, fontFormats[f.fontFormat])
}
func (f *Formatter) styleAttr(styles map[chroma.TokenType]string, tt chroma.TokenType) string {
if _, ok := styles[tt]; !ok {
tt = tt.SubCategory()
if _, ok := styles[tt]; !ok {
tt = tt.Category()
if _, ok := styles[tt]; !ok {
return ""
}
}
}
return styles[tt]
}
func (f *Formatter) styleToSVG(style *chroma.Style) map[chroma.TokenType]string {
converted := map[chroma.TokenType]string{}
bg := style.Get(chroma.Background)
// Convert the style.
for t := range chroma.StandardTypes {
entry := style.Get(t)
if t != chroma.Background {
entry = entry.Sub(bg)
}
if entry.IsZero() {
continue
}
converted[t] = StyleEntryToSVG(entry)
}
return converted
}
// StyleEntryToSVG converts a chroma.StyleEntry to SVG attributes.
func StyleEntryToSVG(e chroma.StyleEntry) string {
var styles []string
if e.Colour.IsSet() {
styles = append(styles, "fill=\""+e.Colour.String()+"\"")
}
if e.Bold == chroma.Yes {
styles = append(styles, "font-weight=\"bold\"")
}
if e.Italic == chroma.Yes {
styles = append(styles, "font-style=\"italic\"")
}
if e.Underline == chroma.Yes {
styles = append(styles, "text-decoration=\"underline\"")
}
return strings.Join(styles, " ")
}

View File

@ -200,6 +200,7 @@ func findClosest(table *ttyTable, seeking chroma.Colour) chroma.Colour {
} }
func styleToEscapeSequence(table *ttyTable, style *chroma.Style) map[chroma.TokenType]string { func styleToEscapeSequence(table *ttyTable, style *chroma.Style) map[chroma.TokenType]string {
style = clearBackground(style)
out := map[chroma.TokenType]string{} out := map[chroma.TokenType]string{}
for _, ttype := range style.Types() { for _, ttype := range style.Types() {
entry := style.Get(ttype) entry := style.Get(ttype)
@ -208,16 +209,22 @@ func styleToEscapeSequence(table *ttyTable, style *chroma.Style) map[chroma.Toke
return out return out
} }
// Clear the background colour.
func clearBackground(style *chroma.Style) *chroma.Style {
builder := style.Builder()
bg := builder.Get(chroma.Background)
bg.Background = 0
bg.NoInherit = true
builder.AddEntry(chroma.Background, bg)
style, _ = builder.Build()
return style
}
type indexedTTYFormatter struct { type indexedTTYFormatter struct {
table *ttyTable table *ttyTable
} }
func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma.Iterator) (err error) { func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma.Iterator) (err error) {
defer func() {
if perr := recover(); perr != nil {
err = perr.(error)
}
}()
theme := styleToEscapeSequence(c.table, style) theme := styleToEscapeSequence(c.table, style)
for token := it(); token != chroma.EOF; token = it() { for token := it(); token != chroma.EOF; token = it() {
// TODO: Cache token lookups? // TODO: Cache token lookups?

View File

@ -11,6 +11,7 @@ import (
var TTY16m = Register("terminal16m", chroma.FormatterFunc(trueColourFormatter)) var TTY16m = Register("terminal16m", chroma.FormatterFunc(trueColourFormatter))
func trueColourFormatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error { func trueColourFormatter(w io.Writer, style *chroma.Style, it chroma.Iterator) error {
style = clearBackground(style)
for token := it(); token != chroma.EOF; token = it() { for token := it(); token != chroma.EOF; token = it() {
entry := style.Get(token.Type) entry := style.Get(token.Type)
if !entry.IsZero() { if !entry.IsZero() {

View File

@ -14,9 +14,12 @@ require (
github.com/gorilla/mux v1.7.3 github.com/gorilla/mux v1.7.3
github.com/mattn/go-colorable v0.0.9 github.com/mattn/go-colorable v0.0.9
github.com/mattn/go-isatty v0.0.4 github.com/mattn/go-isatty v0.0.4
github.com/pkg/errors v0.8.1
github.com/sergi/go-diff v1.0.0 // indirect github.com/sergi/go-diff v1.0.0 // indirect
github.com/stretchr/testify v1.3.0 github.com/stretchr/testify v1.3.0
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
) )
replace github.com/GeertJohan/go.rice => github.com/alecthomas/go.rice v1.0.1-0.20190719113735-961b99d742e7 replace github.com/GeertJohan/go.rice => github.com/alecthomas/go.rice v1.0.1-0.20190719113735-961b99d742e7
go 1.13

View File

@ -10,10 +10,28 @@ Run the tests as normal:
go test ./lexers go test ./lexers
``` ```
## Updating the existing tests ## Update existing tests
When you add a new test data file (`*.actual`), you need to regenerate all tests. That's how Chroma creates the `*.expected` test file based on the corresponding lexer.
You can regenerate all the test outputs To regenerate all tests, type in your terminal:
```go ```go
RECORD=true go test ./lexers RECORD=true go test ./lexers
``` ```
This first sets the `RECORD` environment variable to `true`. Then it runs `go test` on the `./lexers` directory of the Chroma project.
(That environment variable tells Chroma it needs to output test data. After running `go test ./lexers` you can remove or reset that variable.)
### Windows users
Windows users will find that the `RECORD=true go test ./lexers` command fails in both the standard command prompt terminal and in PowerShell.
Instead we have to perform both steps separately:
- Set the `RECORD` environment variable to `true`.
+ In the regular command prompt window, the `set` command sets an environment variable for the current session: `set RECORD=true`. See [this page](https://superuser.com/questions/212150/how-to-set-env-variable-in-windows-cmd-line) for more.
+ In PowerShell, you can use the `$env:RECORD = 'true'` command for that. See [this article](https://mcpmag.com/articles/2019/03/28/environment-variables-in-powershell.aspx) for more.
+ You can also make a persistent environment variable by hand in the Windows computer settings. See [this article](https://www.computerhope.com/issues/ch000549.htm) for how.
- When the environment variable is set, run `go tests ./lexers`.
Chroma will now regenerate the test files and print its results to the console window.

View File

@ -31,7 +31,7 @@ var Applescript = internal.Register(MustNewLexer(
{`\b(as )(alias |application |boolean |class |constant |date |file |integer |list |number |POSIX file |real |record |reference |RGB color |script |text |unit types|(?:Unicode )?text|string)\b`, ByGroups(Keyword, NameClass), nil}, {`\b(as )(alias |application |boolean |class |constant |date |file |integer |list |number |POSIX file |real |record |reference |RGB color |script |text |unit types|(?:Unicode )?text|string)\b`, ByGroups(Keyword, NameClass), nil},
{`\b(AppleScript|current application|false|linefeed|missing value|pi|quote|result|return|space|tab|text item delimiters|true|version)\b`, NameConstant, nil}, {`\b(AppleScript|current application|false|linefeed|missing value|pi|quote|result|return|space|tab|text item delimiters|true|version)\b`, NameConstant, nil},
{`\b(ASCII (character|number)|activate|beep|choose URL|choose application|choose color|choose file( name)?|choose folder|choose from list|choose remote application|clipboard info|close( access)?|copy|count|current date|delay|delete|display (alert|dialog)|do shell script|duplicate|exists|get eof|get volume settings|info for|launch|list (disks|folder)|load script|log|make|mount volume|new|offset|open( (for access|location))?|path to|print|quit|random number|read|round|run( script)?|say|scripting components|set (eof|the clipboard to|volume)|store script|summarize|system attribute|system info|the clipboard|time to GMT|write|quoted form)\b`, NameBuiltin, nil}, {`\b(ASCII (character|number)|activate|beep|choose URL|choose application|choose color|choose file( name)?|choose folder|choose from list|choose remote application|clipboard info|close( access)?|copy|count|current date|delay|delete|display (alert|dialog)|do shell script|duplicate|exists|get eof|get volume settings|info for|launch|list (disks|folder)|load script|log|make|mount volume|new|offset|open( (for access|location))?|path to|print|quit|random number|read|round|run( script)?|say|scripting components|set (eof|the clipboard to|volume)|store script|summarize|system attribute|system info|the clipboard|time to GMT|write|quoted form)\b`, NameBuiltin, nil},
{`\b(considering|else|error|exit|from|if|ignoring|in|repeat|tell|then|times|to|try|until|using terms from|while|whith|with timeout( of)?|with transaction|by|continue|end|its?|me|my|return|of|as)\b`, Keyword, nil}, {`\b(considering|else|error|exit|from|if|ignoring|in|repeat|tell|then|times|to|try|until|using terms from|while|with|with timeout( of)?|with transaction|by|continue|end|its?|me|my|return|of|as)\b`, Keyword, nil},
{`\b(global|local|prop(erty)?|set|get)\b`, Keyword, nil}, {`\b(global|local|prop(erty)?|set|get)\b`, Keyword, nil},
{`\b(but|put|returning|the)\b`, NameBuiltin, nil}, {`\b(but|put|returning|the)\b`, NameBuiltin, nil},
{`\b(attachment|attribute run|character|day|month|paragraph|word|year)s?\b`, NameBuiltin, nil}, {`\b(attachment|attribute run|character|day|month|paragraph|word|year)s?\b`, NameBuiltin, nil},

View File

@ -25,7 +25,7 @@ var Ballerina = internal.Register(MustNewLexer(
{`(annotation|bind|but|endpoint|error|function|object|private|public|returns|service|type|var|with|worker)\b`, KeywordDeclaration, nil}, {`(annotation|bind|but|endpoint|error|function|object|private|public|returns|service|type|var|with|worker)\b`, KeywordDeclaration, nil},
{`(boolean|byte|decimal|float|int|json|map|nil|record|string|table|xml)\b`, KeywordType, nil}, {`(boolean|byte|decimal|float|int|json|map|nil|record|string|table|xml)\b`, KeywordType, nil},
{`(true|false|null)\b`, KeywordConstant, nil}, {`(true|false|null)\b`, KeywordConstant, nil},
{`import(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")}, {`(import)(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")},
{`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, {`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
{`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil}, {`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil},
{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil}, {`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil},

76
vendor/github.com/alecthomas/chroma/lexers/b/bibtex.go generated vendored Normal file
View File

@ -0,0 +1,76 @@
package b
import (
. "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/internal"
)
// Bibtex lexer.
var Bibtex = internal.Register(MustNewLexer(
&Config{
Name: "BibTeX",
Aliases: []string{"bib", "bibtex"},
Filenames: []string{"*.bib"},
MimeTypes: []string{"text/x-bibtex"},
NotMultiline: true,
CaseInsensitive: true,
},
Rules{
"root": {
Include("whitespace"),
{`@comment`, Comment, nil},
{`@preamble`, NameClass, Push("closing-brace", "value", "opening-brace")},
{`@string`, NameClass, Push("closing-brace", "field", "opening-brace")},
{"@[a-z_@!$&*+\\-./:;<>?\\[\\\\\\]^`|~][\\w@!$&*+\\-./:;<>?\\[\\\\\\]^`|~]*", NameClass, Push("closing-brace", "command-body", "opening-brace")},
{`.+`, Comment, nil},
},
"opening-brace": {
Include("whitespace"),
{`[{(]`, Punctuation, Pop(1)},
},
"closing-brace": {
Include("whitespace"),
{`[})]`, Punctuation, Pop(1)},
},
"command-body": {
Include("whitespace"),
{`[^\s\,\}]+`, NameLabel, Push("#pop", "fields")},
},
"fields": {
Include("whitespace"),
{`,`, Punctuation, Push("field")},
Default(Pop(1)),
},
"field": {
Include("whitespace"),
{"[a-z_@!$&*+\\-./:;<>?\\[\\\\\\]^`|~][\\w@!$&*+\\-./:;<>?\\[\\\\\\]^`|~]*", NameAttribute, Push("value", "=")},
Default(Pop(1)),
},
"=": {
Include("whitespace"),
{`=`, Punctuation, Pop(1)},
},
"value": {
Include("whitespace"),
{"[a-z_@!$&*+\\-./:;<>?\\[\\\\\\]^`|~][\\w@!$&*+\\-./:;<>?\\[\\\\\\]^`|~]*", NameVariable, nil},
{`"`, LiteralString, Push("quoted-string")},
{`\{`, LiteralString, Push("braced-string")},
{`[\d]+`, LiteralNumber, nil},
{`#`, Punctuation, nil},
Default(Pop(1)),
},
"quoted-string": {
{`\{`, LiteralString, Push("braced-string")},
{`"`, LiteralString, Pop(1)},
{`[^\{\"]+`, LiteralString, nil},
},
"braced-string": {
{`\{`, LiteralString, Push()},
{`\}`, LiteralString, Pop(1)},
{`[^\{\}]+`, LiteralString, nil},
},
"whitespace": {
{`\s+`, Text, nil},
},
},
))

View File

@ -24,7 +24,8 @@ var CPP = internal.Register(MustNewLexer(
{`(u8|u|U)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")}, {`(u8|u|U)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
{`(L?)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")}, {`(L?)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
{`(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')`, ByGroups(LiteralStringAffix, LiteralStringChar, LiteralStringChar, LiteralStringChar), nil}, {`(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')`, ByGroups(LiteralStringAffix, LiteralStringChar, LiteralStringChar, LiteralStringChar), nil},
{`(\.([0-9](?:'?[0-9]+)*)([eE][+-]?([0-9]('?[0-9]+)*))?|([0-9]('?[0-9]+)*)(([eE][+-]?([0-9]('?[0-9]+)*))|\.([0-9]('?[0-9]+)*)?([eE][+-]?([0-9]('?[0-9]+)*))?)|0[xX](\.([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)([pP][-+]?([0-9]('?[0-9]+)*))?|([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)(([pP][-+]?([0-9]('?[0-9]+)*))|\.([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)?([pP][-+]?([0-9]('?[0-9]+)*))?)))[fFLlUu]*`, LiteralNumberFloat, nil}, {`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
{`0[xX]([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)[LlUu]*`, LiteralNumberHex, nil}, {`0[xX]([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)[LlUu]*`, LiteralNumberHex, nil},
{`0('?[0-7]+)+[LlUu]*`, LiteralNumberOct, nil}, {`0('?[0-7]+)+[LlUu]*`, LiteralNumberOct, nil},
{`0[Bb][01]('?[01]+)*[LlUu]*`, LiteralNumberBin, nil}, {`0[Bb][01]('?[01]+)*[LlUu]*`, LiteralNumberBin, nil},

View File

@ -22,7 +22,7 @@ var Docker = internal.Register(MustNewLexer(
{`(ONBUILD)((?:\s*\\?\s*))`, ByGroups(Keyword, Using(b.Bash)), nil}, {`(ONBUILD)((?:\s*\\?\s*))`, ByGroups(Keyword, Using(b.Bash)), nil},
{`(HEALTHCHECK)(((?:\s*\\?\s*)--\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil}, {`(HEALTHCHECK)(((?:\s*\\?\s*)--\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil},
{`(VOLUME|ENTRYPOINT|CMD|SHELL)((?:\s*\\?\s*))(\[.*?\])`, ByGroups(Keyword, Using(b.Bash), Using(j.JSON)), nil}, {`(VOLUME|ENTRYPOINT|CMD|SHELL)((?:\s*\\?\s*))(\[.*?\])`, ByGroups(Keyword, Using(b.Bash), Using(j.JSON)), nil},
{`(LABEL|ENV|ARG)(((?:\s*\\?\s*)\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil}, {`(LABEL|ENV|ARG)((?:(?:\s*\\?\s*)\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil},
{`((?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)|VOLUME)\b(.*)`, ByGroups(Keyword, LiteralString), nil}, {`((?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)|VOLUME)\b(.*)`, ByGroups(Keyword, LiteralString), nil},
{`((?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY))`, Keyword, nil}, {`((?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY))`, Keyword, nil},
{`(.*\\\n)*.+`, Using(b.Bash), nil}, {`(.*\\\n)*.+`, Using(b.Bash), nil},

View File

@ -38,7 +38,6 @@ func httpContentBlock(groups []string, lexer Lexer) Iterator {
{Generic, groups[0]}, {Generic, groups[0]},
} }
return Literator(tokens...) return Literator(tokens...)
} }
func httpHeaderBlock(groups []string, lexer Lexer) Iterator { func httpHeaderBlock(groups []string, lexer Lexer) Iterator {
@ -66,7 +65,7 @@ func httpBodyContentTypeLexer(lexer Lexer) Lexer { return &httpBodyContentTyper{
type httpBodyContentTyper struct{ Lexer } type httpBodyContentTyper struct{ Lexer }
func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (Iterator, error) { // nolint: gocognit
var contentType string var contentType string
var isContentType bool var isContentType bool
var subIterator Iterator var subIterator Iterator
@ -123,9 +122,7 @@ func (d *httpBodyContentTyper) Tokenise(options *TokeniseOptions, text string) (
return EOF return EOF
} }
} }
} }
return token return token
}, nil }, nil
} }

View File

@ -1,3 +1,4 @@
// Package internal contains common API functions and structures shared between lexer packages.
package internal package internal
import ( import (

73
vendor/github.com/alecthomas/chroma/lexers/j/j.go generated vendored Normal file
View File

@ -0,0 +1,73 @@
package j
import (
. "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/internal"
)
// J lexer.
var J = internal.Register(MustNewLexer(
&Config{
Name: "J",
Aliases: []string{"j"},
Filenames: []string{"*.ijs"},
MimeTypes: []string{"text/x-j"},
},
Rules{
"root": {
{`#!.*$`, CommentPreproc, nil},
{`NB\..*`, CommentSingle, nil},
{`\n+\s*Note`, CommentMultiline, Push("comment")},
{`\s*Note.*`, CommentSingle, nil},
{`\s+`, Text, nil},
{`'`, LiteralString, Push("singlequote")},
{`0\s+:\s*0|noun\s+define\s*$`, NameEntity, Push("nounDefinition")},
{`(([1-4]|13)\s+:\s*0|(adverb|conjunction|dyad|monad|verb)\s+define)\b`, NameFunction, Push("explicitDefinition")},
{Words(``, `\b[a-zA-Z]\w*\.`, `for_`, `goto_`, `label_`), NameLabel, nil},
{Words(``, `\.`, `assert`, `break`, `case`, `catch`, `catchd`, `catcht`, `continue`, `do`, `else`, `elseif`, `end`, `fcase`, `for`, `if`, `return`, `select`, `throw`, `try`, `while`, `whilst`), NameLabel, nil},
{`\b[a-zA-Z]\w*`, NameVariable, nil},
{Words(``, ``, `ARGV`, `CR`, `CRLF`, `DEL`, `Debug`, `EAV`, `EMPTY`, `FF`, `JVERSION`, `LF`, `LF2`, `Note`, `TAB`, `alpha17`, `alpha27`, `apply`, `bind`, `boxopen`, `boxxopen`, `bx`, `clear`, `cutLF`, `cutopen`, `datatype`, `def`, `dfh`, `drop`, `each`, `echo`, `empty`, `erase`, `every`, `evtloop`, `exit`, `expand`, `fetch`, `file2url`, `fixdotdot`, `fliprgb`, `getargs`, `getenv`, `hfd`, `inv`, `inverse`, `iospath`, `isatty`, `isutf8`, `items`, `leaf`, `list`, `nameclass`, `namelist`, `names`, `nc`, `nl`, `on`, `pick`, `rows`, `script`, `scriptd`, `sign`, `sminfo`, `smoutput`, `sort`, `split`, `stderr`, `stdin`, `stdout`, `table`, `take`, `timespacex`, `timex`, `tmoutput`, `toCRLF`, `toHOST`, `toJ`, `tolower`, `toupper`, `type`, `ucp`, `ucpcount`, `usleep`, `utf8`, `uucp`), NameFunction, nil},
{`=[.:]`, Operator, nil},
{"[-=+*#$%@!~`^&\";:.,<>{}\\[\\]\\\\|/]", Operator, nil},
{`[abCdDeEfHiIjLMoprtT]\.`, KeywordReserved, nil},
{`[aDiLpqsStux]\:`, KeywordReserved, nil},
{`(_[0-9])\:`, KeywordConstant, nil},
{`\(`, Punctuation, Push("parentheses")},
Include("numbers"),
},
"comment": {
{`[^)]`, CommentMultiline, nil},
{`^\)`, CommentMultiline, Pop(1)},
{`[)]`, CommentMultiline, nil},
},
"explicitDefinition": {
{`\b[nmuvxy]\b`, NameDecorator, nil},
Include("root"),
{`[^)]`, Name, nil},
{`^\)`, NameLabel, Pop(1)},
{`[)]`, Name, nil},
},
"numbers": {
{`\b_{1,2}\b`, LiteralNumber, nil},
{`_?\d+(\.\d+)?(\s*[ejr]\s*)_?\d+(\.?=\d+)?`, LiteralNumber, nil},
{`_?\d+\.(?=\d+)`, LiteralNumberFloat, nil},
{`_?\d+x`, LiteralNumberIntegerLong, nil},
{`_?\d+`, LiteralNumberInteger, nil},
},
"nounDefinition": {
{`[^)]`, LiteralString, nil},
{`^\)`, NameLabel, Pop(1)},
{`[)]`, LiteralString, nil},
},
"parentheses": {
{`\)`, Punctuation, Pop(1)},
Include("explicitDefinition"),
Include("root"),
},
"singlequote": {
{`[^']`, LiteralString, nil},
{`''`, LiteralString, nil},
{`'`, LiteralString, Pop(1)},
},
},
))

View File

@ -16,6 +16,7 @@ var Nim = internal.Register(MustNewLexer(
}, },
Rules{ Rules{
"root": { "root": {
{`#\[[\s\S]*?\]#`, CommentMultiline, nil},
{`##.*$`, LiteralStringDoc, nil}, {`##.*$`, LiteralStringDoc, nil},
{`#.*$`, Comment, nil}, {`#.*$`, Comment, nil},
{`[*=><+\-/@$~&%!?|\\\[\]]`, Operator, nil}, {`[*=><+\-/@$~&%!?|\\\[\]]`, Operator, nil},

File diff suppressed because one or more lines are too long

View File

@ -35,8 +35,13 @@ func ByGroups(emitters ...Emitter) Emitter {
return EmitterFunc(func(groups []string, lexer Lexer) Iterator { return EmitterFunc(func(groups []string, lexer Lexer) Iterator {
iterators := make([]Iterator, 0, len(groups)-1) iterators := make([]Iterator, 0, len(groups)-1)
// NOTE: If this panics, there is a mismatch with groups // NOTE: If this panics, there is a mismatch with groups
for i, group := range groups[1:] { if len(emitters) != len(groups)-1 {
iterators = append(iterators, emitters[i].Emit([]string{group}, lexer)) iterators = append(iterators, Error.Emit(groups, lexer))
// panic(errors.Errorf("number of groups %q does not match number of emitters %v", groups, emitters))
} else {
for i, group := range groups[1:] {
iterators = append(iterators, emitters[i].Emit([]string{group}, lexer))
}
} }
return Concaterator(iterators...) return Concaterator(iterators...)
}) })
@ -255,7 +260,7 @@ func (l *LexerState) Get(key interface{}) interface{} {
} }
// Iterator returns the next Token from the lexer. // Iterator returns the next Token from the lexer.
func (l *LexerState) Iterator() Token { func (l *LexerState) Iterator() Token { // nolint: gocognit
for l.Pos < len(l.Text) && len(l.Stack) > 0 { for l.Pos < len(l.Text) && len(l.Stack) > 0 {
// Exhaust the iterator stack, if any. // Exhaust the iterator stack, if any.
for len(l.iteratorStack) > 0 { for len(l.iteratorStack) > 0 {

View File

@ -66,7 +66,6 @@ func TypeRemappingLexer(lexer Lexer, mapping TypeMapping) Lexer {
km[k] = rt.To km[k] = rt.To
} }
} }
} }
return RemappingLexer(lexer, func(t Token) []Token { return RemappingLexer(lexer, func(t Token) []Token {
if k, ok := lut[t.Type]; ok { if k, ok := lut[t.Type]; ok {

5
vendor/modules.txt vendored
View File

@ -1,7 +1,8 @@
# github.com/alecthomas/chroma v0.6.7 # github.com/alecthomas/chroma v0.6.8
github.com/alecthomas/chroma github.com/alecthomas/chroma
github.com/alecthomas/chroma/formatters github.com/alecthomas/chroma/formatters
github.com/alecthomas/chroma/formatters/html github.com/alecthomas/chroma/formatters/html
github.com/alecthomas/chroma/formatters/svg
github.com/alecthomas/chroma/lexers github.com/alecthomas/chroma/lexers
github.com/alecthomas/chroma/lexers/a github.com/alecthomas/chroma/lexers/a
github.com/alecthomas/chroma/lexers/b github.com/alecthomas/chroma/lexers/b
@ -48,8 +49,6 @@ github.com/mattn/go-isatty
github.com/mgutz/ansi github.com/mgutz/ansi
# github.com/mitchellh/go-homedir v1.1.0 # github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/go-homedir github.com/mitchellh/go-homedir
# github.com/tj/front v0.0.0-20170212063142-739be213b0a1
github.com/tj/front
# golang.org/x/sys v0.0.0-20191008105621-543471e840be # golang.org/x/sys v0.0.0-20191008105621-543471e840be
golang.org/x/sys/unix golang.org/x/sys/unix
# gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 # gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0