Implementing a Mergable Programmatic Lexer, How to Define Scopes and Transition between States?

Hello,

I apologize for the delayed reply.  We never received an email notification for this post and just noticed it now.  Here's how we do scopes with our C# language in the .NET Languages Add-on:

// In lexer constructor:

ProgrammaticLexicalState docCommentState = new ProgrammaticLexicalState(CSharpLexicalStateId.DocumentationComment, "Documentation comment");
docCommentState.LexicalScopes.Add(new ProgrammaticLexicalScope(new ProgrammaticLexicalScopeMatch(this.IsDocumentationCommentStateScopeStart), new ProgrammaticLexicalScopeMatch(this.IsDocumentationCommentStateScopeEnd)));
this.LexicalStates.Add(docCommentState);
defaultState.ChildLexicalStates.Add(docCommentState);

// Scope methods:

private MergableLexerResult IsDocumentationCommentStateScopeStart(ITextBufferReader reader, ILexicalScope lexicalScope) {
	if (reader.Peek() == '/') {
		reader.Read();
		if (reader.Peek() == '/') {
			reader.Read();
			if (reader.Peek() == '/') {
				reader.Read();
				return new MergableLexerResult(MatchType.ExactMatch, new LexicalScopeTokenData(lexicalScope, CSharpTokenId.DocumentationCommentDelimiter));
			}
			reader.ReadReverse();
		}
		reader.ReadReverse();
	}
	return MergableLexerResult.NoMatch;
}

private MergableLexerResult IsDocumentationCommentStateScopeEnd(ITextBufferReader reader, ILexicalScope lexicalScope) {
	if (Array.IndexOf(lineTerminatorChars, reader.Peek()) != -1) {
		reader.Read();
		return new MergableLexerResult(MatchType.ExactMatch, new LexicalScopeTokenData(lexicalScope, CSharpTokenId.LineTerminator));
	}
	return MergableLexerResult.NoMatch;
}

This is a bit more complex than yours will be but it should help.