improved parsing performance (71a56c25) · Commits · MTS / TTCN-3 / T3Q

de.ugoe.cs.swe.T3Q/src/de/ugoe/cs/swe/T3Q/FileParser.java

0 → 100644

+54 −0

Original line number	Diff line number	Diff line
		package de.ugoe.cs.swe.T3Q;

		import java.io.File;
		import java.io.IOException;
		import java.util.concurrent.Callable;

		import org.eclipse.emf.common.util.URI;
		import org.eclipse.emf.ecore.resource.Resource;
		import org.eclipse.xtext.resource.XtextResource;
		import org.eclipse.xtext.resource.XtextResourceSet;

		import de.ugoe.cs.swe.common.MiscTools;
		import de.ugoe.cs.swe.validation.TTCN3StatisticsProvider;

		public class FileParser implements Callable<Resource> {
		private final File file;

		public FileParser(final File file) {
		this.file = file;
		}

		private Resource parseFile() {
		final XtextResourceSet resourceSet = new XtextResourceSet();
		resourceSet.addLoadOption(XtextResource.OPTION_RESOLVE_ALL, Boolean.FALSE);

		final long millisStart = System.currentTimeMillis();

		Resource res;
		int lines = 0;
		try {
		lines = T3Q.countLines(file.getPath());
		} catch (IOException e) {
		e.printStackTrace();
		}
		TTCN3StatisticsProvider.getInstance().addParsedLines(lines);

		String path = file.getAbsolutePath();
		res = resourceSet.getResource(URI.createURI("file:///" + path), true);

		final long millisEnd = System.currentTimeMillis();

		final long millis = millisEnd - millisStart;

		System.out.println("Parsing file: " + file.getPath() + " (LOC: " + lines + ")" + '\n' +" ...done in "
		+ millis + "ms (" + MiscTools.msToString(millis) + " minutes).");

		return res;
		}

		@Override
		public Resource call() throws Exception {
		return parseFile();
		}
		}

de.ugoe.cs.swe.T3Q/src/de/ugoe/cs/swe/T3Q/TTCN3ResourceProvider.java

+57 −59

Original line number	Diff line number	Diff line
		@@ -5,7 +5,13 @@ import java.io.IOException;
		import java.util.ArrayList;
		import java.util.Collection;
		import java.util.HashMap;
		import java.util.List;
		import java.util.Map;
		import java.util.Set;
		import java.util.concurrent.ExecutionException;
		import java.util.concurrent.ExecutorService;
		import java.util.concurrent.Executors;
		import java.util.concurrent.Future;
		import java.util.concurrent.TimeUnit;
		import java.util.regex.Matcher;
		import java.util.regex.Pattern;
		@@ -24,11 +30,12 @@ import org.eclipse.emf.ecore.util.Diagnostician;
		import org.eclipse.emf.ecore.util.EcoreUtil;
		import org.eclipse.emf.ecore.xmi.impl.XMIResourceFactoryImpl;
		import org.eclipse.emf.ecore.xmi.impl.XMIResourceImpl;
		import org.eclipse.xtext.resource.SynchronizedXtextResourceSet;
		import org.eclipse.xtext.resource.XtextResource;
		import org.eclipse.xtext.resource.XtextResourceSet;
		import org.eclipse.xtext.validation.FeatureBasedDiagnostic;

		import com.google.common.base.Stopwatch;
		import com.google.common.collect.Sets;
		import com.google.inject.Inject;
		import com.google.inject.Injector;

		@@ -38,6 +45,7 @@ import de.ugoe.cs.swe.common.MiscTools;
		import de.ugoe.cs.swe.common.logging.LoggingInterface;
		import de.ugoe.cs.swe.common.logging.LoggingInterface.LogLevel;
		import de.ugoe.cs.swe.common.logging.LoggingInterface.MessageClass;
		import de.ugoe.cs.swe.scoping.TTCN3GlobalScopeProvider;
		import de.ugoe.cs.swe.validation.TTCN3StatisticsProvider;

		public class TTCN3ResourceProvider {
		@@ -45,9 +53,6 @@ public class TTCN3ResourceProvider {
		private LoggingInterface logger;
		private QualityCheckProfile activeProfile;

		@Inject
		private XtextResourceSet resourceSet;

		private final Stopwatch stopwatch = Stopwatch.createUnstarted();

		public TTCN3ResourceProvider(final ArrayList<String> paths,
		@@ -63,66 +68,60 @@ public class TTCN3ResourceProvider {
		Injector injector = new TTCN3StandaloneSetup()
		.createInjectorAndDoEMFRegistration();
		injector.injectMembers(this);
		resourceSet.addLoadOption(XtextResource.OPTION_RESOLVE_ALL,
		Boolean.TRUE);

		stopwatch.start();

		ExecutorService pool = Executors.newFixedThreadPool(4);
		ArrayList<FileParser> parser = new ArrayList<FileParser>();


		for (String path : paths) {
		File file = new File(path);

		// check if file is a ignored one and if so do not parse it
		Pattern pattern = Pattern.compile(activeProfile.getIgnoredResourceRegExp());
		Matcher matcher = null;

		if (file.isFile()) {
		parseFile(file);
		matcher = pattern.matcher(file.getName());
		if(!matcher.matches()) {
		parser.add(new FileParser(file));
		}
		} else if (file.isDirectory()) {
		Collection<File> files = FileUtils.listFiles(file,
		new RegexFileFilter("^.ttcn3$\|^.ttcn$\|^.*3mp$"),
		DirectoryFileFilter.DIRECTORY);

		for (File f : files) {
		parseFile(f);
		matcher = pattern.matcher(f.getName());
		if(!matcher.matches()) {
		parser.add(new FileParser(f));
		}
		}
		}

		TTCN3StatisticsProvider.getInstance().setParsingCompleted(true);
		System.out.println("Test suite parsed in "
		+ MiscTools.msToString(TTCN3StatisticsProvider
		.getInstance().getParseTime()) + " minutes.");
		}

		private void parseFile(File f) {
		// check if file is a ignored one and if so do not parse it
		Pattern pattern = Pattern.compile(activeProfile.getIgnoredResourceRegExp());
		Matcher matcher = pattern.matcher(f.getName());
		if(matcher.matches()) { return; }

		int lines = 0;
		try {
		lines = T3Q.countLines(f.getPath());
		} catch (IOException e) {
		List<Future<Resource>> list = pool.invokeAll(parser);
		for (Future<Resource> f : list) {
		Resource r = f.get();
		TTCN3GlobalScopeProvider.RESOURCES.add(r);
		}
		pool.shutdown();
		} catch (InterruptedException \| ExecutionException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
		}
		TTCN3StatisticsProvider.getInstance().addParsedLines(lines);

		System.out.println("Parsing file: " + f.getPath() + " (LOC: " + lines
		+ ") ...");
		stopwatch.start();
		String path = f.getAbsolutePath();
		resourceSet.getResource(URI.createURI("file:///" + path), true);
		stopwatch.stop();

		System.out
		.println(" ...done in "
		+ stopwatch.elapsed(TimeUnit.MILLISECONDS)
		+ "ms ("
		+ MiscTools.secondsToString(stopwatch
		.elapsed(TimeUnit.SECONDS)) + " minutes).");

		TTCN3StatisticsProvider.getInstance().addParseTime(
		stopwatch.elapsed(TimeUnit.MILLISECONDS));
		stopwatch.reset();
		TTCN3StatisticsProvider.getInstance().setParsingCompleted(true);
		System.out.println("Test suite parsed in "
		+ MiscTools.msToString(stopwatch.elapsed(TimeUnit.MILLISECONDS)) + " minutes.");
		}

		public void analyzeResources() {
		for (Resource r : resourceSet.getResources()) {
		Stopwatch watchAnalyzing = Stopwatch.createStarted();
		for (Resource r : TTCN3GlobalScopeProvider.RESOURCES) {
		System.out.println("Analyzing file: "
		+ r.getURI().devicePath().replaceFirst("///", "") + " ...");

		@@ -130,16 +129,16 @@ public class TTCN3ResourceProvider {
		EcoreUtil.resolveAll(r);
		validate(r);
		// TODO: make this configurable for debug purposes
		// for (Diagnostic d : r.getErrors()) {
		// TTCN3StatisticsProvider.getInstance().incrementCountUniversal();
		// logger.logInformation(r.getURI(), d.getLine(), d.getLine(),
		// MessageClass.UNIVERSAL, d.getMessage());
		// }
		// for (Diagnostic d : r.getWarnings()) {
		// TTCN3StatisticsProvider.getInstance().incrementCountUniversal();
		// logger.logInformation(r.getURI(), d.getLine(), d.getLine(),
		// MessageClass.UNIVERSAL, d.getMessage());
		// }
		for (Diagnostic d : r.getErrors()) {
		TTCN3StatisticsProvider.getInstance().incrementCountUniversal();
		logger.logInformation(r.getURI(), d.getLine(), d.getLine(),
		MessageClass.UNIVERSAL, d.getMessage());
		}
		for (Diagnostic d : r.getWarnings()) {
		TTCN3StatisticsProvider.getInstance().incrementCountUniversal();
		logger.logInformation(r.getURI(), d.getLine(), d.getLine(),
		MessageClass.UNIVERSAL, d.getMessage());
		}

		stopwatch.stop();

		@@ -175,8 +174,7 @@ public class TTCN3ResourceProvider {
		}

		System.out.println("Test suite analyzed in "
		+ MiscTools.msToString(TTCN3StatisticsProvider
		.getInstance().getAnalyzeTime()) + " minutes." + '\n');
		+ MiscTools.msToString(watchAnalyzing.elapsed(TimeUnit.MILLISECONDS)) + " minutes." + '\n');
		}

		private void validate(Resource resource) {
		@@ -287,7 +285,7 @@ public class TTCN3ResourceProvider {
		}

		public void createXMI() {
		for (Resource r : resourceSet.getResources()) {
		for (Resource r : TTCN3GlobalScopeProvider.RESOURCES) {
		String extension = fileExtensionOrPath(r.getURI().devicePath(),
		true) + "m";
		storeResourceContents(r.getContents(), r.getURI().devicePath()

de.ugoe.cs.swe.TTCN3/src/de/ugoe/cs/swe/scoping/TTCN3GlobalScopeProvider.java

+7 −5

Original line number	Diff line number	Diff line
		@@ -43,6 +43,8 @@ public class TTCN3GlobalScopeProvider extends AbstractGlobalScopeProvider {
		private static final SetMultimap<Resource, ImportData> imports = HashMultimap.create();
		private static final SetMultimap<TTCN3Module, TTCN3Module> importedFrom = HashMultimap.create();

		public static final Set<Resource> RESOURCES = Sets.newHashSet();

		@Inject
		private IResourceScopeCache cache = IResourceScopeCache.NullImpl.INSTANCE;

		@@ -71,9 +73,9 @@ public class TTCN3GlobalScopeProvider extends AbstractGlobalScopeProvider {

		if (TTCN3StatisticsProvider.getInstance().isParsingCompleted() && staticScope.containsKey(resource)) {
		return staticScope.get(resource);
		} else /if (TTCN3StatisticsProvider.getInstance().isParsingCompleted()) / {
		} else if (TTCN3StatisticsProvider.getInstance().isParsingCompleted()) {
		if (!importsString.isEmpty()) {
		for (Resource r : resource.getResourceSet().getResources()) {
		for (Resource r : RESOURCES) {
		ISelectable descriptions = getAllDescriptions(r);
		IEObjectDescription module = descriptions.getExportedObjectsByType(TTCN3Package.eINSTANCE.getTTCN3Module()).iterator()
		.next();
		@@ -170,8 +172,8 @@ public class TTCN3GlobalScopeProvider extends AbstractGlobalScopeProvider {
		staticScope.put(resource, scope);
		}
		return scope;
		// } else {
		// return IScope.NULLSCOPE;
		} else {
		return IScope.NULLSCOPE;
		}
		}