Update genproto and antlr.

This commit is contained in:
cici37
2022-05-04 18:37:30 -07:00
parent a86dd29157
commit e8f6184d86
61 changed files with 1070 additions and 1033 deletions

View File

@@ -251,7 +251,7 @@ func (l *LexerATNConfig) hash() int {
f = 0
}
h := murmurInit(7)
h = murmurUpdate(h, l.state.hash())
h = murmurUpdate(h, l.state.GetStateNumber())
h = murmurUpdate(h, l.alt)
h = murmurUpdate(h, l.context.hash())
h = murmurUpdate(h, l.semanticContext.hash())

View File

@@ -11,7 +11,7 @@ type ATNConfigSet interface {
Add(ATNConfig, *DoubleDict) bool
AddAll([]ATNConfig) bool
GetStates() *Set
GetStates() Set
GetPredicates() []SemanticContext
GetItems() []ATNConfig
@@ -35,6 +35,8 @@ type ATNConfigSet interface {
GetConflictingAlts() *BitSet
SetConflictingAlts(*BitSet)
Alts() *BitSet
FullContext() bool
GetUniqueAlt() int
@@ -55,7 +57,7 @@ type BaseATNConfigSet struct {
// effectively doubles the number of objects associated with ATNConfigs. All
// keys are hashed by (s, i, _, pi), not including the context. Wiped out when
// read-only because a set becomes a DFA state.
configLookup *Set
configLookup Set
// configs is the added elements.
configs []ATNConfig
@@ -91,11 +93,19 @@ type BaseATNConfigSet struct {
uniqueAlt int
}
func (b *BaseATNConfigSet) Alts() *BitSet {
alts := NewBitSet()
for _, it := range b.configs {
alts.add(it.GetAlt())
}
return alts
}
func NewBaseATNConfigSet(fullCtx bool) *BaseATNConfigSet {
return &BaseATNConfigSet{
cachedHash: -1,
configLookup: NewSet(nil, equalATNConfigs),
fullCtx: fullCtx,
cachedHash: -1,
configLookup: NewArray2DHashSetWithCap(hashATNConfig, equalATNConfigs, 16, 2),
fullCtx: fullCtx,
}
}
@@ -116,12 +126,11 @@ func (b *BaseATNConfigSet) Add(config ATNConfig, mergeCache *DoubleDict) bool {
b.dipsIntoOuterContext = true
}
existing := b.configLookup.add(config).(ATNConfig)
existing := b.configLookup.Add(config).(ATNConfig)
if existing == config {
b.cachedHash = -1
b.configs = append(b.configs, config) // Track order here
return true
}
@@ -145,11 +154,11 @@ func (b *BaseATNConfigSet) Add(config ATNConfig, mergeCache *DoubleDict) bool {
return true
}
func (b *BaseATNConfigSet) GetStates() *Set {
states := NewSet(nil, nil)
func (b *BaseATNConfigSet) GetStates() Set {
states := NewArray2DHashSet(nil, nil)
for i := 0; i < len(b.configs); i++ {
states.add(b.configs[i].GetState())
states.Add(b.configs[i].GetState())
}
return states
@@ -186,7 +195,7 @@ func (b *BaseATNConfigSet) OptimizeConfigs(interpreter *BaseATNSimulator) {
panic("set is read-only")
}
if b.configLookup.length() == 0 {
if b.configLookup.Len() == 0 {
return
}
@@ -236,13 +245,11 @@ func (b *BaseATNConfigSet) hash() int {
}
func (b *BaseATNConfigSet) hashCodeConfigs() int {
h := murmurInit(1)
for _, c := range b.configs {
if c != nil {
h = murmurUpdate(h, c.hash())
}
h := 1
for _, config := range b.configs {
h = 31*h + config.hash()
}
return murmurFinish(h, len(b.configs))
return h
}
func (b *BaseATNConfigSet) Length() int {
@@ -258,7 +265,7 @@ func (b *BaseATNConfigSet) Contains(item ATNConfig) bool {
panic("not implemented for read-only sets")
}
return b.configLookup.contains(item)
return b.configLookup.Contains(item)
}
func (b *BaseATNConfigSet) ContainsFast(item ATNConfig) bool {
@@ -266,7 +273,7 @@ func (b *BaseATNConfigSet) ContainsFast(item ATNConfig) bool {
panic("not implemented for read-only sets")
}
return b.configLookup.contains(item) // TODO: containsFast is not implemented for Set
return b.configLookup.Contains(item) // TODO: containsFast is not implemented for Set
}
func (b *BaseATNConfigSet) Clear() {
@@ -276,7 +283,7 @@ func (b *BaseATNConfigSet) Clear() {
b.configs = make([]ATNConfig, 0)
b.cachedHash = -1
b.configLookup = NewSet(nil, equalATNConfigs)
b.configLookup = NewArray2DHashSet(nil, equalATNConfigs)
}
func (b *BaseATNConfigSet) FullContext() bool {
@@ -358,11 +365,20 @@ type OrderedATNConfigSet struct {
func NewOrderedATNConfigSet() *OrderedATNConfigSet {
b := NewBaseATNConfigSet(false)
b.configLookup = NewSet(nil, nil)
b.configLookup = NewArray2DHashSet(nil, nil)
return &OrderedATNConfigSet{BaseATNConfigSet: b}
}
func hashATNConfig(i interface{}) int {
o := i.(ATNConfig)
hash := 7
hash = 31*hash + o.GetState().GetStateNumber()
hash = 31*hash + o.GetAlt()
hash = 31*hash + o.GetSemanticContext().hash()
return hash
}
func equalATNConfigs(a, b interface{}) bool {
if a == nil || b == nil {
return false
@@ -379,9 +395,13 @@ func equalATNConfigs(a, b interface{}) bool {
return false
}
nums := ai.GetState().GetStateNumber() == bi.GetState().GetStateNumber()
alts := ai.GetAlt() == bi.GetAlt()
cons := ai.GetSemanticContext().equals(bi.GetSemanticContext())
if ai.GetState().GetStateNumber() != bi.GetState().GetStateNumber() {
return false
}
return nums && alts && cons
if ai.GetAlt() != bi.GetAlt() {
return false
}
return ai.GetSemanticContext().equals(bi.GetSemanticContext())
}

View File

@@ -7,6 +7,7 @@ package antlr
import (
"fmt"
"strconv"
"strings"
)
// DFASerializer is a DFA walker that knows how to dump them to serialized
@@ -112,7 +113,12 @@ func NewLexerDFASerializer(dfa *DFA) *LexerDFASerializer {
}
func (l *LexerDFASerializer) getEdgeLabel(i int) string {
return "'" + string(i) + "'"
var sb strings.Builder
sb.Grow(6)
sb.WriteByte('\'')
sb.WriteRune(rune(i))
sb.WriteByte('\'')
return sb.String()
}
func (l *LexerDFASerializer) String() string {

View File

@@ -50,8 +50,8 @@ type DFAState struct {
// edges elements point to the target of the symbol. Shift up by 1 so (-1)
// Token.EOF maps to the first element.
edges []*DFAState
edgesMu sync.RWMutex
edges []*DFAState
edgesMu sync.RWMutex
isAcceptState bool
@@ -92,16 +92,16 @@ func NewDFAState(stateNumber int, configs ATNConfigSet) *DFAState {
}
// GetAltSet gets the set of all alts mentioned by all ATN configurations in d.
func (d *DFAState) GetAltSet() *Set {
alts := NewSet(nil, nil)
func (d *DFAState) GetAltSet() Set {
alts := NewArray2DHashSet(nil, nil)
if d.configs != nil {
for _, c := range d.configs.GetItems() {
alts.add(c.GetAlt())
alts.Add(c.GetAlt())
}
}
if alts.length() == 0 {
if alts.Len() == 0 {
return nil
}
@@ -173,26 +173,11 @@ func (d *DFAState) String() string {
}
}
return fmt.Sprintf("%d:%s%s", fmt.Sprint(d.configs), s)
return fmt.Sprintf("%d:%s%s", d.stateNumber, fmt.Sprint(d.configs), s)
}
func (d *DFAState) hash() int {
h := murmurInit(11)
c := 1
if d.isAcceptState {
if d.predicates != nil {
for _, p := range d.predicates {
h = murmurUpdate(h, p.alt)
h = murmurUpdate(h, p.pred.hash())
c += 2
}
} else {
h = murmurUpdate(h, d.prediction)
c += 1
}
}
h := murmurInit(7)
h = murmurUpdate(h, d.configs.hash())
return murmurFinish(h, c)
}
return murmurFinish(h, 1)
}

View File

@@ -226,16 +226,28 @@ func (i *IntervalSet) StringVerbose(literalNames []string, symbolicNames []strin
func (i *IntervalSet) toCharString() string {
names := make([]string, len(i.intervals))
var sb strings.Builder
for j := 0; j < len(i.intervals); j++ {
v := i.intervals[j]
if v.Stop == v.Start+1 {
if v.Start == TokenEOF {
names = append(names, "<EOF>")
} else {
names = append(names, ("'" + string(v.Start) + "'"))
sb.WriteByte('\'')
sb.WriteRune(rune(v.Start))
sb.WriteByte('\'')
names = append(names, sb.String())
sb.Reset()
}
} else {
names = append(names, "'"+string(v.Start)+"'..'"+string(v.Stop-1)+"'")
sb.WriteByte('\'')
sb.WriteRune(rune(v.Start))
sb.WriteString("'..'")
sb.WriteRune(rune(v.Stop - 1))
sb.WriteByte('\'')
names = append(names, sb.String())
sb.Reset()
}
}
if len(names) > 1 {

View File

@@ -414,10 +414,9 @@ func (l *LexerIndexedCustomAction) execute(lexer Lexer) {
func (l *LexerIndexedCustomAction) hash() int {
h := murmurInit(0)
h = murmurUpdate(h, l.actionType)
h = murmurUpdate(h, l.offset)
h = murmurUpdate(h, l.lexerAction.hash())
return murmurFinish(h, 3)
return murmurFinish(h, 2)
}
func (l *LexerIndexedCustomAction) equals(other LexerAction) bool {

View File

@@ -161,10 +161,13 @@ func (l *LexerActionExecutor) hash() int {
func (l *LexerActionExecutor) equals(other interface{}) bool {
if l == other {
return true
} else if _, ok := other.(*LexerActionExecutor); !ok {
return false
} else {
return l.cachedHash == other.(*LexerActionExecutor).cachedHash &&
&l.lexerActions == &other.(*LexerActionExecutor).lexerActions
}
othert, ok := other.(*LexerActionExecutor)
if !ok {
return false
}
if othert == nil {
return false
}
return l.cachedHash == othert.cachedHash && &l.lexerActions == &othert.lexerActions
}

View File

@@ -7,6 +7,7 @@ package antlr
import (
"fmt"
"strconv"
"strings"
)
var (
@@ -206,7 +207,7 @@ func (l *LexerATNSimulator) getExistingTargetState(s *DFAState, t int) *DFAState
return nil
}
target := s.getIthEdge(t-LexerATNSimulatorMinDFAEdge)
target := s.getIthEdge(t - LexerATNSimulatorMinDFAEdge)
if LexerATNSimulatorDebug && target != nil {
fmt.Println("reuse state " + strconv.Itoa(s.stateNumber) + " edge to " + strconv.Itoa(target.stateNumber))
}
@@ -299,7 +300,7 @@ func (l *LexerATNSimulator) getReachableConfigSet(input CharStream, closure ATNC
func (l *LexerATNSimulator) accept(input CharStream, lexerActionExecutor *LexerActionExecutor, startIndex, index, line, charPos int) {
if LexerATNSimulatorDebug {
fmt.Printf("ACTION %s\n", lexerActionExecutor)
fmt.Printf("ACTION %v\n", lexerActionExecutor)
}
// seek to after last char in token
input.Seek(index)
@@ -630,7 +631,13 @@ func (l *LexerATNSimulator) GetTokenName(tt int) string {
return "EOF"
}
return "'" + string(tt) + "'"
var sb strings.Builder
sb.Grow(6)
sb.WriteByte('\'')
sb.WriteRune(rune(tt))
sb.WriteByte('\'')
return sb.String()
}
func resetSimState(sim *SimState) {

View File

@@ -38,7 +38,7 @@ func (la *LL1Analyzer) getDecisionLookahead(s ATNState) []*IntervalSet {
look := make([]*IntervalSet, count)
for alt := 0; alt < count; alt++ {
look[alt] = NewIntervalSet()
lookBusy := NewSet(nil, nil)
lookBusy := NewArray2DHashSet(nil, nil)
seeThruPreds := false // fail to get lookahead upon pred
la.look1(s.GetTransitions()[alt].getTarget(), nil, BasePredictionContextEMPTY, look[alt], lookBusy, NewBitSet(), seeThruPreds, false)
// Wipe out lookahead for la alternative if we found nothing
@@ -75,7 +75,7 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet
if ctx != nil {
lookContext = predictionContextFromRuleContext(s.GetATN(), ctx)
}
la.look1(s, stopState, lookContext, r, NewSet(nil, nil), NewBitSet(), seeThruPreds, true)
la.look1(s, stopState, lookContext, r, NewArray2DHashSet(nil, nil), NewBitSet(), seeThruPreds, true)
return r
}
@@ -109,22 +109,22 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet
// outermost context is reached. This parameter has no effect if {@code ctx}
// is {@code nil}.
func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) {
func (la *LL1Analyzer) look2(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) {
returnState := la.atn.states[ctx.getReturnState(i)]
la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF)
}
func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool) {
func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool) {
c := NewBaseATNConfig6(s, 0, ctx)
if lookBusy.contains(c) {
if lookBusy.Contains(c) {
return
}
lookBusy.add(c)
lookBusy.Add(c)
if s == stopState {
if ctx == nil {
@@ -148,13 +148,13 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look
}
if ctx != BasePredictionContextEMPTY {
removed := calledRuleStack.contains(s.GetRuleIndex())
defer func() {
if removed {
calledRuleStack.add(s.GetRuleIndex())
}
}()
calledRuleStack.remove(s.GetRuleIndex())
removed := calledRuleStack.contains(s.GetRuleIndex())
defer func() {
if removed {
calledRuleStack.add(s.GetRuleIndex())
}
}()
calledRuleStack.remove(s.GetRuleIndex())
// run thru all possible stack tops in ctx
for i := 0; i < ctx.length(); i++ {
returnState := la.atn.states[ctx.getReturnState(i)]
@@ -198,7 +198,7 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx PredictionContext, look
}
}
func (la *LL1Analyzer) look3(stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy *Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, t1 *RuleTransition) {
func (la *LL1Analyzer) look3(stopState ATNState, ctx PredictionContext, look *IntervalSet, lookBusy Set, calledRuleStack *BitSet, seeThruPreds, addEOF bool, t1 *RuleTransition) {
newContext := SingletonBasePredictionContextCreate(ctx, t1.followState.GetStateNumber())

View File

@@ -15,6 +15,7 @@ var (
ParserATNSimulatorListATNDecisions = false
ParserATNSimulatorDFADebug = false
ParserATNSimulatorRetryDebug = false
TurnOffLRLoopEntryBranchOpt = false
)
type ParserATNSimulator struct {
@@ -135,6 +136,7 @@ func (p *ParserATNSimulator) AdaptivePredict(input TokenStream, decision int, ou
// appropriate start state for the precedence level rather
// than simply setting DFA.s0.
//
dfa.s0.configs = s0Closure
s0Closure = p.applyPrecedenceFilter(s0Closure)
s0 = p.addDFAState(dfa, NewDFAState(-1, s0Closure))
dfa.setPrecedenceStartState(p.parser.GetPrecedence(), s0)
@@ -259,11 +261,13 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream,
stopIndex := input.Index()
input.Seek(startIndex)
alts := p.evalSemanticContext(D.predicates, outerContext, true)
if alts.length() == 0 {
switch alts.length() {
case 0:
panic(p.noViableAlt(input, outerContext, D.configs, startIndex))
} else if alts.length() == 1 {
case 1:
return alts.minValue()
} else {
default:
// Report ambiguity after predicate evaluation to make sure the correct set of ambig alts is Reported.
p.ReportAmbiguity(dfa, D, startIndex, stopIndex, false, alts, D.configs)
return alts.minValue()
@@ -292,11 +296,11 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream,
func (p *ParserATNSimulator) getExistingTargetState(previousD *DFAState, t int) *DFAState {
edges := previousD.getEdges()
if edges == nil {
if edges == nil || t+1 < 0 || t+1 >= len(edges) {
return nil
}
return previousD.getIthEdge(t+1)
return previousD.getIthEdge(t + 1)
}
// Compute a target state for an edge in the DFA, and attempt to add the
@@ -422,7 +426,8 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 AT
if reach.GetUniqueAlt() != ATNInvalidAltNumber {
predictedAlt = reach.GetUniqueAlt()
break
} else if p.predictionMode != PredictionModeLLExactAmbigDetection {
}
if p.predictionMode != PredictionModeLLExactAmbigDetection {
predictedAlt = PredictionModeresolvesToJustOneViableAlt(altSubSets)
if predictedAlt != ATNInvalidAltNumber {
break
@@ -479,7 +484,7 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 AT
// the fact that we should predict alternative 1. We just can't say for
// sure that there is an ambiguity without looking further.
p.ReportAmbiguity(dfa, D, startIndex, input.Index(), foundExactAmbig, nil, reach)
p.ReportAmbiguity(dfa, D, startIndex, input.Index(), foundExactAmbig, reach.Alts(), reach)
return predictedAlt
}
@@ -503,7 +508,7 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
// ensure that the alternative Matching the longest overall sequence is
// chosen when multiple such configurations can Match the input.
var SkippedStopStates []*BaseATNConfig
var skippedStopStates []*BaseATNConfig
// First figure out where we can reach on input t
for _, c := range closure.GetItems() {
@@ -511,14 +516,9 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
fmt.Println("testing " + p.GetTokenName(t) + " at " + c.String())
}
_, ok := c.GetState().(*RuleStopState)
if ok {
if _, ok := c.GetState().(*RuleStopState); ok {
if fullCtx || t == TokenEOF {
if SkippedStopStates == nil {
SkippedStopStates = make([]*BaseATNConfig, 0)
}
SkippedStopStates = append(SkippedStopStates, c.(*BaseATNConfig))
skippedStopStates = append(skippedStopStates, c.(*BaseATNConfig))
if ParserATNSimulatorDebug {
fmt.Println("added " + c.String() + " to SkippedStopStates")
}
@@ -526,8 +526,7 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
continue
}
for j := 0; j < len(c.GetState().GetTransitions()); j++ {
trans := c.GetState().GetTransitions()[j]
for _, trans := range c.GetState().GetTransitions() {
target := p.getReachableTarget(trans, t)
if target != nil {
cfg := NewBaseATNConfig4(c, target)
@@ -538,6 +537,7 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
}
}
}
// Now figure out where the reach operation can take us...
var reach ATNConfigSet
@@ -550,7 +550,7 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
// condition is not true when one or more configurations have been
// withheld in SkippedStopStates, or when the current symbol is EOF.
//
if SkippedStopStates == nil && t != TokenEOF {
if skippedStopStates == nil && t != TokenEOF {
if len(intermediate.configs) == 1 {
// Don't pursue the closure if there is just one state.
// It can only have one alternative just add to result
@@ -568,9 +568,10 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
//
if reach == nil {
reach = NewBaseATNConfigSet(fullCtx)
closureBusy := NewSet(nil, nil)
closureBusy := NewArray2DHashSet(nil, nil)
treatEOFAsEpsilon := t == TokenEOF
for k := 0; k < len(intermediate.configs); k++ {
amount := len(intermediate.configs)
for k := 0; k < amount; k++ {
p.closure(intermediate.configs[k], reach, closureBusy, false, fullCtx, treatEOFAsEpsilon)
}
}
@@ -602,9 +603,9 @@ func (p *ParserATNSimulator) computeReachSet(closure ATNConfigSet, t int, fullCt
// chooses an alternative Matching the longest overall sequence when
// multiple alternatives are viable.
//
if SkippedStopStates != nil && ((!fullCtx) || (!PredictionModehasConfigInRuleStopState(reach))) {
for l := 0; l < len(SkippedStopStates); l++ {
reach.Add(SkippedStopStates[l], p.mergeCache)
if skippedStopStates != nil && ((!fullCtx) || (!PredictionModehasConfigInRuleStopState(reach))) {
for l := 0; l < len(skippedStopStates); l++ {
reach.Add(skippedStopStates[l], p.mergeCache)
}
}
if len(reach.GetItems()) == 0 {
@@ -640,10 +641,7 @@ func (p *ParserATNSimulator) removeAllConfigsNotInRuleStopState(configs ATNConfi
}
result := NewBaseATNConfigSet(configs.FullContext())
for _, config := range configs.GetItems() {
_, ok := config.GetState().(*RuleStopState)
if ok {
if _, ok := config.GetState().(*RuleStopState); ok {
result.Add(config, p.mergeCache)
continue
}
@@ -665,7 +663,7 @@ func (p *ParserATNSimulator) computeStartState(a ATNState, ctx RuleContext, full
for i := 0; i < len(a.GetTransitions()); i++ {
target := a.GetTransitions()[i].getTarget()
c := NewBaseATNConfig6(target, i+1, initialContext)
closureBusy := NewSet(nil, nil)
closureBusy := NewArray2DHashSet(nil, nil)
p.closure(c, configs, closureBusy, true, fullCtx, false)
}
return configs
@@ -787,7 +785,7 @@ func (p *ParserATNSimulator) getPredsForAmbigAlts(ambigAlts *BitSet, configs ATN
}
}
nPredAlts := 0
for i := 1; i < nalts+1; i++ {
for i := 1; i <= nalts; i++ {
pred := altToPred[i]
if pred == nil {
altToPred[i] = SemanticContextNone
@@ -972,14 +970,13 @@ func (p *ParserATNSimulator) evalSemanticContext(predPredictions []*PredPredicti
return predictions
}
func (p *ParserATNSimulator) closure(config ATNConfig, configs ATNConfigSet, closureBusy *Set, collectPredicates, fullCtx, treatEOFAsEpsilon bool) {
func (p *ParserATNSimulator) closure(config ATNConfig, configs ATNConfigSet, closureBusy Set, collectPredicates, fullCtx, treatEOFAsEpsilon bool) {
initialDepth := 0
p.closureCheckingStopState(config, configs, closureBusy, collectPredicates,
fullCtx, initialDepth, treatEOFAsEpsilon)
}
func (p *ParserATNSimulator) closureCheckingStopState(config ATNConfig, configs ATNConfigSet, closureBusy *Set, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) {
func (p *ParserATNSimulator) closureCheckingStopState(config ATNConfig, configs ATNConfigSet, closureBusy Set, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) {
if ParserATNSimulatorDebug {
fmt.Println("closure(" + config.String() + ")")
fmt.Println("configs(" + configs.String() + ")")
@@ -988,8 +985,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config ATNConfig, configs
}
}
_, ok := config.GetState().(*RuleStopState)
if ok {
if _, ok := config.GetState().(*RuleStopState); ok {
// We hit rule end. If we have context info, use it
// run thru all possible stack tops in ctx
if !config.GetContext().isEmpty() {
@@ -1033,7 +1029,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config ATNConfig, configs
}
// Do the actual work of walking epsilon edges//
func (p *ParserATNSimulator) closureWork(config ATNConfig, configs ATNConfigSet, closureBusy *Set, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) {
func (p *ParserATNSimulator) closureWork(config ATNConfig, configs ATNConfigSet, closureBusy Set, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) {
state := config.GetState()
// optimization
if !state.GetEpsilonOnlyTransitions() {
@@ -1042,30 +1038,24 @@ func (p *ParserATNSimulator) closureWork(config ATNConfig, configs ATNConfigSet,
// both epsilon transitions and non-epsilon transitions.
}
for i := 0; i < len(state.GetTransitions()); i++ {
if i == 0 && p.canDropLoopEntryEdgeInLeftRecursiveRule(config) {
continue
}
t := state.GetTransitions()[i]
_, ok := t.(*ActionTransition)
continueCollecting := collectPredicates && !ok
c := p.getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEOFAsEpsilon)
if ci, ok := c.(*BaseATNConfig); ok && ci != nil {
if !t.getIsEpsilon() && closureBusy.add(c) != c {
// avoid infinite recursion for EOF* and EOF+
continue
}
newDepth := depth
if _, ok := config.GetState().(*RuleStopState); ok {
// target fell off end of rule mark resulting c as having dipped into outer context
// We can't get here if incoming config was rule stop and we had context
// track how far we dip into outer context. Might
// come in handy and we avoid evaluating context dependent
// preds if p is > 0.
if closureBusy.add(c) != c {
// avoid infinite recursion for right-recursive rules
continue
}
if p.dfa != nil && p.dfa.getPrecedenceDfa() {
if t.(*EpsilonTransition).outermostPrecedenceReturn == p.dfa.atnStartState.GetRuleIndex() {
c.setPrecedenceFilterSuppressed(true)
@@ -1073,15 +1063,27 @@ func (p *ParserATNSimulator) closureWork(config ATNConfig, configs ATNConfigSet,
}
c.SetReachesIntoOuterContext(c.GetReachesIntoOuterContext() + 1)
if closureBusy.Add(c) != c {
// avoid infinite recursion for right-recursive rules
continue
}
configs.SetDipsIntoOuterContext(true) // TODO: can remove? only care when we add to set per middle of p method
newDepth--
if ParserATNSimulatorDebug {
fmt.Println("dips into outer ctx: " + c.String())
}
} else if _, ok := t.(*RuleTransition); ok {
// latch when newDepth goes negative - once we step out of the entry context we can't return
if newDepth >= 0 {
newDepth++
} else {
if !t.getIsEpsilon() && closureBusy.Add(c) != c {
// avoid infinite recursion for EOF* and EOF+
continue
}
if _, ok := t.(*RuleTransition); ok {
// latch when newDepth goes negative - once we step out of the entry context we can't return
if newDepth >= 0 {
newDepth++
}
}
}
p.closureCheckingStopState(c, configs, closureBusy, continueCollecting, fullCtx, newDepth, treatEOFAsEpsilon)
@@ -1089,12 +1091,93 @@ func (p *ParserATNSimulator) closureWork(config ATNConfig, configs ATNConfigSet,
}
}
func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config ATNConfig) bool {
if TurnOffLRLoopEntryBranchOpt {
return false
}
_p := config.GetState()
// First check to see if we are in StarLoopEntryState generated during
// left-recursion elimination. For efficiency, also check if
// the context has an empty stack case. If so, it would mean
// global FOLLOW so we can't perform optimization
if startLoop, ok := _p.(StarLoopEntryState); !ok || !startLoop.precedenceRuleDecision || config.GetContext().isEmpty() || config.GetContext().hasEmptyPath() {
return false
}
// Require all return states to return back to the same rule
// that p is in.
numCtxs := config.GetContext().length()
for i := 0; i < numCtxs; i++ {
returnState := p.atn.states[config.GetContext().getReturnState(i)]
if returnState.GetRuleIndex() != _p.GetRuleIndex() {
return false
}
}
decisionStartState := _p.(BlockStartState).GetTransitions()[0].getTarget().(BlockStartState)
blockEndStateNum := decisionStartState.getEndState().stateNumber
blockEndState := p.atn.states[blockEndStateNum].(*BlockEndState)
// Verify that the top of each stack context leads to loop entry/exit
// state through epsilon edges and w/o leaving rule.
for i := 0; i < numCtxs; i++ { // for each stack context
returnStateNumber := config.GetContext().getReturnState(i)
returnState := p.atn.states[returnStateNumber]
// all states must have single outgoing epsilon edge
if len(returnState.GetTransitions()) != 1 || !returnState.GetTransitions()[0].getIsEpsilon() {
return false
}
// Look for prefix op case like 'not expr', (' type ')' expr
returnStateTarget := returnState.GetTransitions()[0].getTarget()
if returnState.GetStateType() == ATNStateBlockEnd && returnStateTarget == _p {
continue
}
// Look for 'expr op expr' or case where expr's return state is block end
// of (...)* internal block; the block end points to loop back
// which points to p but we don't need to check that
if returnState == blockEndState {
continue
}
// Look for ternary expr ? expr : expr. The return state points at block end,
// which points at loop entry state
if returnStateTarget == blockEndState {
continue
}
// Look for complex prefix 'between expr and expr' case where 2nd expr's
// return state points at block end state of (...)* internal block
if returnStateTarget.GetStateType() == ATNStateBlockEnd &&
len(returnStateTarget.GetTransitions()) == 1 &&
returnStateTarget.GetTransitions()[0].getIsEpsilon() &&
returnStateTarget.GetTransitions()[0].getTarget() == _p {
continue
}
// anything else ain't conforming
return false
}
return true
}
func (p *ParserATNSimulator) getRuleName(index int) string {
if p.parser != nil && index >= 0 {
return p.parser.GetRuleNames()[index]
}
var sb strings.Builder
sb.Grow(32)
return "<rule " + fmt.Sprint(index) + ">"
sb.WriteString("<rule ")
sb.WriteString(strconv.FormatInt(int64(index), 10))
sb.WriteByte('>')
return sb.String()
}
func (p *ParserATNSimulator) getEpsilonTarget(config ATNConfig, t Transition, collectPredicates, inContext, fullCtx, treatEOFAsEpsilon bool) ATNConfig {
@@ -1110,25 +1193,7 @@ func (p *ParserATNSimulator) getEpsilonTarget(config ATNConfig, t Transition, co
return p.actionTransition(config, t.(*ActionTransition))
case TransitionEPSILON:
return NewBaseATNConfig4(config, t.getTarget())
case TransitionATOM:
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if treatEOFAsEpsilon {
if t.Matches(TokenEOF, 0, 1) {
return NewBaseATNConfig4(config, t.getTarget())
}
}
return nil
case TransitionRANGE:
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if treatEOFAsEpsilon {
if t.Matches(TokenEOF, 0, 1) {
return NewBaseATNConfig4(config, t.getTarget())
}
}
return nil
case TransitionSET:
case TransitionATOM, TransitionRANGE, TransitionSET:
// EOF transitions act like epsilon transitions after the first EOF
// transition is traversed
if treatEOFAsEpsilon {
@@ -1196,7 +1261,7 @@ func (p *ParserATNSimulator) predTransition(config ATNConfig, pt *PredicateTrans
}
}
var c *BaseATNConfig
if collectPredicates && ((pt.isCtxDependent && inContext) || !pt.isCtxDependent) {
if collectPredicates && (!pt.isCtxDependent || inContext) {
if fullCtx {
// In full context mode, we can evaluate predicates on-the-fly
// during closure, which dramatically reduces the size of

View File

@@ -58,9 +58,15 @@ func calculateHash(parent PredictionContext, returnState int) int {
return murmurFinish(h, 2)
}
var _emptyPredictionContextHash int
func init() {
_emptyPredictionContextHash = murmurInit(1)
_emptyPredictionContextHash = murmurFinish(_emptyPredictionContextHash, 0)
}
func calculateEmptyHash() int {
h := murmurInit(1)
return murmurFinish(h, 0)
return _emptyPredictionContextHash
}
// Used to cache {@link BasePredictionContext} objects. Its used for the shared
@@ -113,15 +119,15 @@ type BaseSingletonPredictionContext struct {
}
func NewBaseSingletonPredictionContext(parent PredictionContext, returnState int) *BaseSingletonPredictionContext {
var cachedHash int
if parent != nil {
cachedHash = calculateHash(parent, returnState)
} else {
cachedHash = calculateEmptyHash()
}
s := new(BaseSingletonPredictionContext)
s.BasePredictionContext = NewBasePredictionContext(37)
if parent != nil {
s.cachedHash = calculateHash(parent, returnState)
} else {
s.cachedHash = calculateEmptyHash()
}
s.BasePredictionContext = NewBasePredictionContext(cachedHash)
s.parentCtx = parent
s.returnState = returnState
@@ -175,15 +181,7 @@ func (b *BaseSingletonPredictionContext) equals(other PredictionContext) bool {
}
func (b *BaseSingletonPredictionContext) hash() int {
h := murmurInit(1)
if b.parentCtx == nil {
return murmurFinish(h, 0)
}
h = murmurUpdate(h, b.parentCtx.hash())
h = murmurUpdate(h, b.returnState)
return murmurFinish(h, 2)
return b.cachedHash
}
func (b *BaseSingletonPredictionContext) String() string {
@@ -253,13 +251,20 @@ func NewArrayPredictionContext(parents []PredictionContext, returnStates []int)
// from {@link //EMPTY} and non-empty. We merge {@link //EMPTY} by using
// nil parent and
// returnState == {@link //EmptyReturnState}.
hash := murmurInit(1)
for _, parent := range parents {
hash = murmurUpdate(hash, parent.hash())
}
for _, returnState := range returnStates {
hash = murmurUpdate(hash, returnState)
}
hash = murmurFinish(hash, len(parents)<<1)
c := new(ArrayPredictionContext)
c.BasePredictionContext = NewBasePredictionContext(37)
for i := range parents {
c.cachedHash += calculateHash(parents[i], returnStates[i])
}
c.BasePredictionContext = NewBasePredictionContext(hash)
c.parents = parents
c.returnStates = returnStates
@@ -305,17 +310,7 @@ func (a *ArrayPredictionContext) equals(other PredictionContext) bool {
}
func (a *ArrayPredictionContext) hash() int {
h := murmurInit(1)
for _, p := range a.parents {
h = murmurUpdate(h, p.hash())
}
for _, r := range a.returnStates {
h = murmurUpdate(h, r)
}
return murmurFinish(h, 2 * len(a.parents))
return a.BasePredictionContext.cachedHash
}
func (a *ArrayPredictionContext) String() string {

View File

@@ -49,7 +49,7 @@ var tokenTypeMapCache = make(map[string]int)
var ruleIndexMapCache = make(map[string]int)
func (b *BaseRecognizer) checkVersion(toolVersion string) {
runtimeVersion := "4.9.2"
runtimeVersion := "4.9.3"
if runtimeVersion != toolVersion {
fmt.Println("ANTLR runtime and generated code versions disagree: " + runtimeVersion + "!=" + toolVersion)
}

View File

@@ -108,7 +108,15 @@ func (p *Predicate) equals(other interface{}) bool {
}
func (p *Predicate) hash() int {
return p.ruleIndex*43 + p.predIndex*47
h := murmurInit(0)
h = murmurUpdate(h, p.ruleIndex)
h = murmurUpdate(h, p.predIndex)
if p.isCtxDependent {
h = murmurUpdate(h, 1)
} else {
h = murmurUpdate(h, 0)
}
return murmurFinish(h, 3)
}
func (p *Predicate) String() string {
@@ -154,21 +162,24 @@ func (p *PrecedencePredicate) equals(other interface{}) bool {
}
func (p *PrecedencePredicate) hash() int {
return p.precedence * 51
h := uint32(1)
h = 31*h + uint32(p.precedence)
return int(h)
}
func (p *PrecedencePredicate) String() string {
return "{" + strconv.Itoa(p.precedence) + ">=prec}?"
}
func PrecedencePredicatefilterPrecedencePredicates(set *Set) []*PrecedencePredicate {
func PrecedencePredicatefilterPrecedencePredicates(set Set) []*PrecedencePredicate {
result := make([]*PrecedencePredicate, 0)
for _, v := range set.values() {
set.Each(func(v interface{}) bool {
if c2, ok := v.(*PrecedencePredicate); ok {
result = append(result, c2)
}
}
return true
})
return result
}
@@ -182,21 +193,21 @@ type AND struct {
func NewAND(a, b SemanticContext) *AND {
operands := NewSet(nil, nil)
operands := NewArray2DHashSet(nil, nil)
if aa, ok := a.(*AND); ok {
for _, o := range aa.opnds {
operands.add(o)
operands.Add(o)
}
} else {
operands.add(a)
operands.Add(a)
}
if ba, ok := b.(*AND); ok {
for _, o := range ba.opnds {
operands.add(o)
operands.Add(o)
}
} else {
operands.add(b)
operands.Add(b)
}
precedencePredicates := PrecedencePredicatefilterPrecedencePredicates(operands)
if len(precedencePredicates) > 0 {
@@ -209,10 +220,10 @@ func NewAND(a, b SemanticContext) *AND {
}
}
operands.add(reduced)
operands.Add(reduced)
}
vs := operands.values()
vs := operands.Values()
opnds := make([]SemanticContext, len(vs))
for i, v := range vs {
opnds[i] = v.(SemanticContext)
@@ -334,21 +345,21 @@ type OR struct {
func NewOR(a, b SemanticContext) *OR {
operands := NewSet(nil, nil)
operands := NewArray2DHashSet(nil, nil)
if aa, ok := a.(*OR); ok {
for _, o := range aa.opnds {
operands.add(o)
operands.Add(o)
}
} else {
operands.add(a)
operands.Add(a)
}
if ba, ok := b.(*OR); ok {
for _, o := range ba.opnds {
operands.add(o)
operands.Add(o)
}
} else {
operands.add(b)
operands.Add(b)
}
precedencePredicates := PrecedencePredicatefilterPrecedencePredicates(operands)
if len(precedencePredicates) > 0 {
@@ -361,10 +372,10 @@ func NewOR(a, b SemanticContext) *OR {
}
}
operands.add(reduced)
operands.Add(reduced)
}
vs := operands.values()
vs := operands.Values()
opnds := make([]SemanticContext, len(vs))
for i, v := range vs {

View File

@@ -7,6 +7,7 @@ package antlr
import (
"fmt"
"strconv"
"strings"
)
// atom, set, epsilon, action, predicate, rule transitions.
@@ -236,7 +237,13 @@ func (t *RangeTransition) Matches(symbol, minVocabSymbol, maxVocabSymbol int) bo
}
func (t *RangeTransition) String() string {
return "'" + string(t.start) + "'..'" + string(t.stop) + "'"
var sb strings.Builder
sb.WriteByte('\'')
sb.WriteRune(rune(t.start))
sb.WriteString("'..'")
sb.WriteRune(rune(t.stop))
sb.WriteByte('\'')
return sb.String()
}
type AbstractPredicateTransition interface {

View File

@@ -47,35 +47,6 @@ func (s *IntStack) Push(e int) {
*s = append(*s, e)
}
type Set struct {
data map[int][]interface{}
hashcodeFunction func(interface{}) int
equalsFunction func(interface{}, interface{}) bool
}
func NewSet(
hashcodeFunction func(interface{}) int,
equalsFunction func(interface{}, interface{}) bool) *Set {
s := new(Set)
s.data = make(map[int][]interface{})
if hashcodeFunction != nil {
s.hashcodeFunction = hashcodeFunction
} else {
s.hashcodeFunction = standardHashFunction
}
if equalsFunction == nil {
s.equalsFunction = standardEqualsFunction
} else {
s.equalsFunction = equalsFunction
}
return s
}
func standardEqualsFunction(a interface{}, b interface{}) bool {
ac, oka := a.(comparable)
@@ -100,72 +71,6 @@ type hasher interface {
hash() int
}
func (s *Set) length() int {
return len(s.data)
}
func (s *Set) add(value interface{}) interface{} {
key := s.hashcodeFunction(value)
values := s.data[key]
if s.data[key] != nil {
for i := 0; i < len(values); i++ {
if s.equalsFunction(value, values[i]) {
return values[i]
}
}
s.data[key] = append(s.data[key], value)
return value
}
v := make([]interface{}, 1, 10)
v[0] = value
s.data[key] = v
return value
}
func (s *Set) contains(value interface{}) bool {
key := s.hashcodeFunction(value)
values := s.data[key]
if s.data[key] != nil {
for i := 0; i < len(values); i++ {
if s.equalsFunction(value, values[i]) {
return true
}
}
}
return false
}
func (s *Set) values() []interface{} {
var l []interface{}
for _, v := range s.data {
l = append(l, v...)
}
return l
}
func (s *Set) String() string {
r := ""
for _, av := range s.data {
for _, v := range av {
r += fmt.Sprint(v)
}
}
return r
}
type BitSet struct {
data map[int]bool
}
@@ -353,65 +258,38 @@ func PrintArrayJavaStyle(sa []string) string {
return buffer.String()
}
// The following routines were lifted from bits.rotate* available in Go 1.9.
const uintSize = 32 << (^uint(0) >> 32 & 1) // 32 or 64
// rotateLeft returns the value of x rotated left by (k mod UintSize) bits.
// To rotate x right by k bits, call RotateLeft(x, -k).
func rotateLeft(x uint, k int) uint {
if uintSize == 32 {
return uint(rotateLeft32(uint32(x), k))
}
return uint(rotateLeft64(uint64(x), k))
}
// rotateLeft32 returns the value of x rotated left by (k mod 32) bits.
func rotateLeft32(x uint32, k int) uint32 {
const n = 32
s := uint(k) & (n - 1)
return x<<s | x>>(n-s)
}
// rotateLeft64 returns the value of x rotated left by (k mod 64) bits.
func rotateLeft64(x uint64, k int) uint64 {
const n = 64
s := uint(k) & (n - 1)
return x<<s | x>>(n-s)
}
// murmur hash
const (
c1_32 uint = 0xCC9E2D51
c2_32 uint = 0x1B873593
n1_32 uint = 0xE6546B64
)
func murmurInit(seed int) int {
return seed
}
func murmurUpdate(h1 int, k1 int) int {
var k1u uint
k1u = uint(k1) * c1_32
k1u = rotateLeft(k1u, 15)
k1u *= c2_32
func murmurUpdate(h int, value int) int {
const c1 uint32 = 0xCC9E2D51
const c2 uint32 = 0x1B873593
const r1 uint32 = 15
const r2 uint32 = 13
const m uint32 = 5
const n uint32 = 0xE6546B64
var h1u = uint(h1) ^ k1u
k1u = rotateLeft(k1u, 13)
h1u = h1u*5 + 0xe6546b64
return int(h1u)
k := uint32(value)
k *= c1
k = (k << r1) | (k >> (32 - r1))
k *= c2
hash := uint32(h) ^ k
hash = (hash << r2) | (hash >> (32 - r2))
hash = hash*m + n
return int(hash)
}
func murmurFinish(h1 int, numberOfWords int) int {
var h1u uint = uint(h1)
h1u ^= uint(numberOfWords * 4)
h1u ^= h1u >> 16
h1u *= uint(0x85ebca6b)
h1u ^= h1u >> 13
h1u *= 0xc2b2ae35
h1u ^= h1u >> 16
func murmurFinish(h int, numberOfWords int) int {
var hash = uint32(h)
hash ^= uint32(numberOfWords) << 2
hash ^= hash >> 16
hash *= 0x85ebca6b
hash ^= hash >> 13
hash *= 0xc2b2ae35
hash ^= hash >> 16
return int(h1u)
return int(hash)
}