antlr / codebuff Goto Github PK

        catch (MalformedURLException mue) {
            errMgr.runTimeError(null, null, ErrorType.INVALID_TEMPLATE_NAME,
                                mue, parent); // this is aligned not indented so column delta should be -4 not -24.
        }
        return null;

System.out.printf("classify calls %d, hits %d rate %f\n",
                  kNNClassifier.nClassifyCalls, kNNClassifier.nClassifyCacheHits,
                  kNNClassifier.nClassifyCacheHits/(float)kNNClassifier.nClassifyCalls);

/*
 * Copyright 2019 Peter <[email protected]>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package hk.quantr.antlrformatter;

import java.io.File;
import java.util.List;
import java.util.Map;
import org.antlr.codebuff.Corpus;
import org.antlr.codebuff.Formatter;
import org.antlr.codebuff.InputDocument;
import static org.antlr.codebuff.Tool.format;
import static org.antlr.codebuff.Tool.getFilenames;
import static org.antlr.codebuff.Tool.getLexer;
import static org.antlr.codebuff.Tool.load;
import static org.antlr.codebuff.Tool.parse;
import static org.antlr.codebuff.Trainer.FEATURES_HPOS;
import static org.antlr.codebuff.Trainer.FEATURES_INJECT_WS;
import static org.antlr.codebuff.misc.BuffUtils.filter;
import org.antlr.codebuff.misc.LangDescriptor;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.misc.Utils;
import org.apache.commons.io.IOUtils;
import org.junit.Test;

/**
 *
 * @author Peter <[email protected]>
 */
public class Test1 {

	@Test
	public void test() throws Exception {
//		String s = IOUtils.toString(AntlrFormatter.class.getResourceAsStream("Hello.g4"), "utf8");
//		String s = IOUtils.toString(AntlrFormatter.class.getResourceAsStream("AssemblerLexer.g4"), "utf8");
		String s = IOUtils.toString(AntlrFormatter.class.getResourceAsStream("AssemblerParser.g4"), "utf8");
//		System.out.println(AntlrFormatter.format(s).substring(0, 10000));

		String grammarName = "org.antlr.codebuff.ANTLRv4";
		String startRule = "grammarSpec";
		String fileExtension = "g4";
		String fileRegex = ".*\\." + fileExtension;
		String corpusDir = "corpus/antlr4/training";
		int indentSize = 4;
		int singleLineCommentType = -1;
		String commentS = "LINE_COMMENT";
		String parserClassName = grammarName + "Parser";
		String lexerClassName = grammarName + "Lexer";
		Class<? extends Parser> parserClass = null;
		Class<? extends Lexer> lexerClass = null;
		Lexer lexer = null;

		try {
			parserClass = (Class<? extends Parser>) Class.forName(parserClassName);
			lexerClass = (Class<? extends Lexer>) Class.forName(lexerClassName);
		} catch (Exception e) {
			System.err.println("Can't load " + parserClassName + " or maybe " + lexerClassName);
			System.err.println("Make sure they are generated by ANTLR, compiled, and in CLASSPATH");
			e.printStackTrace(System.err);
		}
		if (commentS != null) {
			try {
				lexer = getLexer(lexerClass, null);
			} catch (Exception e) {
				System.err.println("Can't instantiate lexer " + lexerClassName);
				e.printStackTrace(System.err);
			}
			if (lexer == null) {
				return;
			}
			Map<String, Integer> tokenTypeMap = lexer.getTokenTypeMap();
			if (tokenTypeMap.containsKey(commentS)) {
				singleLineCommentType = tokenTypeMap.get(commentS);
			}
		}
		LangDescriptor language = new LangDescriptor(grammarName, corpusDir, fileRegex,
				lexerClass, parserClass, startRule,
				indentSize, singleLineCommentType);
		format(language, "/Users/peter/workspace/antlr-formatter/src/test/resources/hk/quantr/antlrformatter/AssemblerParser.g4", "/Users/peter/Desktop/AssemblerParser.g4");
	}

	public static void format(LangDescriptor language,
			String testFileName,
			String outputFileName)
			throws Exception {
		// load all files up front
		List<String> allFiles = getFilenames(new File(language.corpusDir), language.fileRegex);
		List<InputDocument> documents = load(allFiles, language);
		// if in corpus, don't include in corpus
		final String path = new File(testFileName).getAbsolutePath();
		List<InputDocument> others = filter(documents, d -> !d.fileName.equals(path));
		InputDocument testDoc = parse(testFileName, language);
		Corpus corpus = new Corpus(others, language);
		corpus.train();

		Formatter formatter = new Formatter(corpus, language.indentSize, Formatter.DEFAULT_K,
				FEATURES_INJECT_WS, FEATURES_HPOS);
		String output = formatter.format(testDoc, false);

		if (outputFileName != null) {
			Utils.writeFile(outputFileName, output);
		} else {
			System.out.print(output);
		}
	}
}

-------------------------------------------------------
 T E S T S
-------------------------------------------------------
Running hk.quantr.antlrformatter.Test1
Tests run: 1, Failures: 1, Errors: 0, Skipped: 0, Time elapsed: 0.587 sec <<< FAILURE!
test(hk.quantr.antlrformatter.Test1)  Time elapsed: 0.521 sec  <<< FAILURE!
java.lang.AssertionError
	at org.antlr.v4.runtime.BufferedTokenStream.sync(BufferedTokenStream.java:148)
	at org.antlr.v4.runtime.BufferedTokenStream.previousTokenOnChannel(BufferedTokenStream.java:343)
	at org.antlr.codebuff.misc.CodeBuffTokenStream.getPreviousRealToken(CodeBuffTokenStream.java:45)
	at org.antlr.codebuff.Formatter.getFeatures(Formatter.java:348)
	at org.antlr.codebuff.Formatter.processToken(Formatter.java:189)
	at org.antlr.codebuff.Formatter.format(Formatter.java:161)
	at hk.quantr.antlrformatter.Test1.format(Test1.java:111)
	at hk.quantr.antlrformatter.Test1.test(Test1.java:92)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
	at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:252)
	at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:141)
	at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:112)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at org.apache.maven.surefire.util.ReflectionUtils.invokeMethodWithArray(ReflectionUtils.java:189)
	at org.apache.maven.surefire.booter.ProviderFactory$ProviderProxy.invoke(ProviderFactory.java:165)
	at org.apache.maven.surefire.booter.ProviderFactory.invokeProvider(ProviderFactory.java:85)
	at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:115)
	at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:75)


Results :

Failed tests:   test(hk.quantr.antlrformatter.Test1)

Tests run: 1, Failures: 1, Errors: 0, Skipped: 0

------------------------------------------------------------------------
BUILD FAILURE
------------------------------------------------------------------------
Total time:  4.607 s
Finished at: 2019-10-13T15:46:58+08:00
------------------------------------------------------------------------
Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:2.12.4:test (default-cli) on project antlr-formatter: There are test failures.

Please refer to /Users/peter/workspace/antlr-formatter/target/surefire-reports for the individual test results.
-> [Help 1]

To see the full stack trace of the errors, re-run Maven with the -e switch.
Re-run Maven using the -X switch to enable full debug logging.

For more information about the errors and possible solutions, please read the following articles:
[Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException

grammar Hello;

r : 'hello' ID;

ID    : [a-z]+ ;
WS  : [ \t\r\n]+ -> skip ;

/Users/peter/workspace/antlr-formatter/src/test/resources/hk/quantr/antlrformatter>java -jar ~/workspace/codebuff/target/codebuff-1.5.1.jar -g org.antlr.codebuff.ANTLRv4        -rule grammarSpec        -corpus corpus/antlr4/training        -files g4        -indent 4        -comment LINE_COMMENT Hello.g4
grammar Hello;r:'hello'ID;ID:[a-z]+;WS:[ \t\r\n]+->skip;

x = f(a,
        b,
        c);

should detect alignment at b and keep it until after `)`. otherwise only a,b will be aligned.

                 LT(-1)     |              Pair  Strt      LT(1)        ancestor                     parent                     parent^2                    parent^3                    parent^4                        char
   LT(-1)    right ancestor |    LT(1)     dif\n line  left ancestor  child index      parent     child index     parent^2    child index     parent^3    child index     parent^4    child index       file       line pos 
    (1)           (1)       |     (1)       (1)   (1)       (1)           (1)           (1)           (1)           (1)           (1)           (1)           (1)           (1)           (1)            (0)       (0)  (0) 
============ ============== | ============ ===== ===== ============== ============ ============== ============ ============== ============ ============== ============ ============== ============ =============== ==== ====
   WHERE     id_expr*sion:2 | DELIMITED_ID       true  conditi*pper:1      0       where_clause:1      1        query_block:1      3       subquer*ents:1      0           subquery:1      0       condition15.sql  0       ->{0=1, 30=1}
   WHERE     id_expr*sion:2 | DELIMITED_ID       true  conditi*pper:1      0       where_clause:1      1        query_block:1      3       subquer*ents:1      0           subquery:1      0       cast_multise...  7    5   (cat=30|0|0,d=0.000):      ·"a3"."m_id"="a2"."m_id"
   WHERE     id_expr*sion:2 | DELIMITED_ID       false conditi*pper:1      0       where_clause:1      1        query_block:1      3       subquer*ents:1      0           subquery:1      0         bindvar04.sql  6    10  (cat=0|0|0,d=0.067):     where ·"rm".a-interval:"sys_b_07" day(:"sys_b_08") to second(:"sys_b_09")