Commit 9977ca86 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

ready for testing

parent a2c97f0b
......@@ -138,6 +138,11 @@
<artifactId>jena-tdb</artifactId>
<version>3.13.0</version>
</dependency>
<dependency>
<groupId>fr.emse.ci</groupId>
<artifactId>sparql-generate-xml</artifactId>
<version>${sparql-generate.version}</version>
</dependency>
<dependency>
<groupId>fr.emse.ci</groupId>
<artifactId>sparql-generate-markdown</artifactId>
......@@ -182,6 +187,7 @@
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>fr.emse.gitlab.saref.Main
......
......@@ -27,13 +27,16 @@ public class CMDConfigurations {
public static final String ARG_DIRECTORY_MAN = "Location of the SAREF extension directory (default is .)";
public static final String ARG_INCLUDE_ALL = "a";
public static final String ARG_INCLUDE_ALL_LONG = "all";
public static final String ARG_INCLUDE_ALL_MAN = "Include all branches (not for production)";
public static final String ARG_INCLUDE_ALL_MAN = "Include all branches";
public static final String ARG_INCLUDE_MASTER = "m";
public static final String ARG_INCLUDE_MASTER_LONG = "master";
public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches (not for production)";
public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches";
public static final String ARG_PRODUCTION = "p";
public static final String ARG_PRODUCTION_LONG = "prod";
public static final String ARG_PRODUCTION_MAN = "Production mode";
public static final String ARG_PRODUCTION_LONG = "production";
public static final String ARG_PRODUCTION_MAN = "Production mode: the directory does not contain a SAREF extension, but only a file `.saref-repositories.yml`";
public static final String ARG_DEBUG_TEMPLATE = "debug";
public static final String ARG_DEBUG_TEMPLATE_LONG = "debug-template";
public static final String ARG_DEBUG_TEMPLATE_MAN = "Use the debug-template mode of SPARQL-Generate when generating the documentation";
public static CommandLine parseArguments(String[] args) throws ParseException {
DefaultParser commandLineParser = new DefaultParser();
......@@ -47,7 +50,8 @@ public class CMDConfigurations {
.addOption(ARG_DIRECTORY, ARG_DIRECTORY_LONG, true, ARG_DIRECTORY_MAN)
.addOption(ARG_INCLUDE_MASTER, ARG_INCLUDE_MASTER_LONG, false, ARG_INCLUDE_MASTER_MAN)
.addOption(ARG_INCLUDE_ALL, ARG_INCLUDE_ALL_LONG, false, ARG_INCLUDE_ALL_MAN)
.addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN);
.addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN)
.addOption(ARG_DEBUG_TEMPLATE, ARG_DEBUG_TEMPLATE_LONG, false, ARG_DEBUG_TEMPLATE_MAN);
}
public static void displayHelp() {
......
package fr.emse.gitlab.saref;
import java.io.File;
import java.nio.file.FileSystems;
import java.nio.file.PathMatcher;
import java.util.regex.Pattern;
public class Constants {
public final static String BASE = "https://saref.etsi.org/";
public final static String BASE_DOC = BASE + "documentation/";
public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger";
public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})";
public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)";
public static final String REGEX_MASTER_BRANCH = "^refs/remotes/origin/master$";
public static final String MASTER_BRANCH = "refs/remotes/origin/master";
public static final String REGEX_RELEASE_BRANCH = "^refs/remotes/origin/release-" + REGEX_VERSION + "$";
public static final String REGEX_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$";
public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH);
......@@ -27,7 +32,6 @@ public class Constants {
// output
public static final String TARGET_DIR = "target";
public static final String GIT_DIR = TARGET_DIR + File.separator + "sources";
public static final String DATASET_DIR = TARGET_DIR + File.separator + "tdb";
public static final String LOG_FILE_NAME = TARGET_DIR + File.separator + "output.log";
public static final String SITE_DIR = TARGET_DIR + File.separator + "site";
......@@ -37,9 +41,15 @@ public class Constants {
public static final String SAREF_ACCESS_USERNAME = "SAREF_ACCESS_USERNAME";
public static final String propertyFile = ".saref-repositories";
public static final String CONFIG = BASE + "config";
public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl");
public static boolean INCLUDE_MASTER = false;
public static boolean INCLUDE_ALL = false;
public static boolean PRODUCTION = false;
public static boolean DEBUG_TEMPLATE = false;
}
package fr.emse.gitlab.saref;
import static fr.emse.gitlab.saref.CMDConfigurations.*;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DEBUG_TEMPLATE;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_ALL;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_PRODUCTION;
import java.awt.Desktop;
import java.io.File;
import java.io.FileReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.Map;
import java.util.function.Consumer;
import javax.net.ssl.HttpsURLConnection;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jena.atlas.io.IndentedWriter;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.ReadWrite;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.ResIterator;
import org.apache.jena.sparql.core.Var;
import org.apache.jena.sparql.engine.binding.Binding;
import org.apache.jena.sparql.engine.binding.BindingHashMap;
import org.apache.jena.sparql.util.Context;
import org.apache.jena.tdb.TDBFactory;
import org.apache.jena.vocabulary.RDF;
import org.apache.log4j.Layout;
import org.apache.log4j.PatternLayout;
import org.semanticweb.owlapi.model.OWLOntologyIRIMapper;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.errors.CheckoutConflictException;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.InvalidRefNameException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.eclipse.jgit.api.errors.RefNotFoundException;
import org.eclipse.jgit.revwalk.RevCommit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.ci.sparqlext.SPARQLExt;
import fr.emse.ci.sparqlext.engine.PlanFactory;
import fr.emse.ci.sparqlext.engine.RootPlan;
import fr.emse.ci.sparqlext.stream.LocationMapperAccept;
import fr.emse.ci.sparqlext.stream.LocatorFileAccept;
import fr.emse.ci.sparqlext.stream.LookUpRequest;
import fr.emse.ci.sparqlext.stream.SPARQLExtStreamManager;
import fr.emse.ci.sparqlext.utils.ContextUtils;
import fr.emse.ci.sparqlext.utils.VarUtils;
import fr.emse.gitlab.saref.entities.git.Repositories;
import fr.emse.gitlab.saref.entities.git.Repository;
import fr.emse.gitlab.saref.entities.git.Version;
import fr.emse.gitlab.saref.entities.tests.TestSuites;
import fr.emse.gitlab.saref.jobs.CreateDataset;
import fr.emse.gitlab.saref.jobs.JobRunner;
import fr.emse.gitlab.saref.jobs.ReadRepositories;
import fr.emse.gitlab.saref.jobs.WriteRDFFiles;
import fr.emse.gitlab.saref.jobs.CheckConfig;
import fr.emse.gitlab.saref.jobs.CheckOWLProfile;
import fr.emse.gitlab.saref.jobs.CheckRepositoryStructure;
import fr.emse.gitlab.saref.jobs.CheckShapes;
import fr.emse.gitlab.saref.jobs.CopyStaticFiles;
import fr.emse.gitlab.saref.jobs.CheckoutJob;
import fr.emse.gitlab.saref.jobs.ReadExamples;
import fr.emse.gitlab.saref.jobs.ReadOntology;
import fr.emse.gitlab.saref.jobs.ReadRepositories;
import fr.emse.gitlab.saref.jobs.GeneratePortal;
import fr.emse.gitlab.saref.utils.TestSuitesAppender;
import fr.emse.gitlab.saref.vocabs.EX;
public class Main {
static final Logger LOG = LoggerFactory.getLogger(Main.class);
private static final Layout LAYOUT = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n");
private static final org.apache.log4j.Logger ROOT_LOGGER = org.apache.log4j.Logger.getRootLogger();
static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_BASE + ".Main");
private static TestSuites testSuites = new TestSuites("SAREF pipeline");
private static final Map<String, String> testSuiteNames = new HashMap<>();
private static TestSuites testSuites = new TestSuites();
private static File directory;
private static File target;
private static boolean openBrowser = false;
private static SPARQLExtStreamManager streamManager;
private static boolean openBrowser = true;
private static Dataset dataset;
private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites");
public static void main(String[] args)
throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException {
throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException {
parseCommandArguments(args);
setLogAppenders();
prepareDirectory();
streamManager = initializeStreamManager();
dataset = createFreshDataset();
LOG.info("Starting pipeline in " + directory);
Repositories repositories = readRepositories();
accept(repositories, Main::testRepository);
checkOWLProfile();
new CheckConfig().doJob(dataset);
new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory);
accept(repositories, Main::resetCheckout);
testSuites.prune();
reportAndExit(0);
}
private static void parseCommandArguments(String[] args) throws ParseException, IOException {
CommandLine cl = CMDConfigurations.parseArguments(args);
if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {
CMDConfigurations.displayHelp();
......@@ -73,67 +130,216 @@ public class Main {
Constants.INCLUDE_ALL = true;
}
if (cl.hasOption(ARG_PRODUCTION)) {
if(cl.hasOption(ARG_INCLUDE_MASTER) || cl.hasOption(ARG_INCLUDE_ALL)) {
throw new RuntimeException("Option --production is incompatible with options --master or --all ");
}
Constants.PRODUCTION = true;
}
if (cl.hasOption(ARG_DEBUG_TEMPLATE)) {
Constants.DEBUG_TEMPLATE = true;
}
String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);
if (dirName.equals("")) {
dirName = ARG_DIRECTORY_DEFAULT;
}
directory = new File(dirName).getCanonicalFile();
}
directory = new File(dirName).getAbsoluteFile();
private static void setLogAppenders() throws IOException {
File logFile = new File(directory, Constants.LOG_FILE_NAME);
Layout layout = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n");
org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger();
rootLogger.addAppender(new org.apache.log4j.RollingFileAppender(layout, logFile.getAbsolutePath(), false));
org.apache.log4j.Logger loggerBase = org.apache.log4j.Logger.getLogger(Constants.LOGGER_BASE);
TestSuitesAppender appender = new TestSuitesAppender(testSuites);
loggerBase.addAppender(appender);
}
private static void prepareDirectory() throws IOException {
target = new File(directory, Constants.TARGET_DIR);
FileUtils.forceMkdir(target);
File siteDir = new File(directory, Constants.SITE_DIR);
File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR);
FileUtils.forceMkdir(siteDir);
FileUtils.forceMkdir(staticTargetDir);
File staticDir = new File(Main.class.getClassLoader().getResource("static").getFile());
FileUtils.copyDirectory(staticDir, staticTargetDir);
}
File logFile = new File(directory, Constants.LOG_FILE_NAME);
ROOT_LOGGER.addAppender(new org.apache.log4j.RollingFileAppender(LAYOUT, logFile.getAbsolutePath(), false));
LOG.info("Starting pipeline");
doJob(new CheckRepositoryStructure(directory));
// doJob(new CheckShapes(directory));
// Repositories repositories = doJob(new ReadRepositories(directory));
// Repositories repositories = new Repositories();
// Dataset dataset = doJob(new CreateDataset(directory, repositories));
// doJob(new CopyStaticFiles(directory));
// Set<OWLOntologyIRIMapper> mappers = doJob(new WriteRDFFiles(directory, dataset));
// doJob(new CheckOWLProfile(directory, mappers));
reportAndExit(0);
private static Dataset createFreshDataset() throws IOException {
FileUtils.forceMkdir(new File(directory, Constants.DATASET_DIR));
dataset = TDBFactory.createDataset(Constants.DATASET_DIR);
dataset.begin(ReadWrite.WRITE);
dataset.getDefaultModel().removeAll();
for (Iterator<String> it = dataset.listNames(); it.hasNext();) {
String name = it.next();
dataset.removeNamedModel(name);
}
dataset.addNamedModel(Constants.CONFIG, ModelFactory.createDefaultModel());
dataset.commit();
return dataset;
}
private static void accept(Repositories repositories, Consumer<Repository> consumer) {
if (repositories.getDefaultRepository() != null) {
Repository repository = repositories.getDefaultRepository();
consumer.accept(repository);
}
for(Repository repository : repositories.getNamedRepositories()) {
consumer.accept(repository);
}
}
private static Repositories readRepositories() {
try {
return new ReadRepositories("Fetching required repositories").readRepositories(directory);
} catch (Exception ex) {
reportAndExit(-1);
return null;
}
}
private static void testRepository(Repository repository) {
for (Version version : repository.getVersions()) {
try {
String versionTestSuiteName = version.toString() + " - testing repository structure";
new CheckoutJob(versionTestSuiteName).checkoutVersion(version);
new CheckRepositoryStructure(versionTestSuiteName).check(version);
} catch(SAREFPipelineException ex) {
LOG.error("Error while testing repository structure " + version, ex);
continue;
}
try {
String ontologyTestSuiteName = version.toString() + " - testing ontology file";
testSuiteNames.put(version.getUri(), ontologyTestSuiteName);
new ReadOntology(ontologyTestSuiteName).doJob(dataset, version);
} catch(SAREFPipelineException ex) {
LOG.error(version.toString() + " Found errors for " + version.toString()
+ ". This version and all the examples will be ignored.", ex);
continue;
}
File examplesDir = new File(version.getRepository().getDirectory(), "examples");
try {
for (Iterator<Path> it = Files.walk(examplesDir.toPath()).filter(p -> {
return Constants.ttlMatcher.matches(p);
}).iterator();it.hasNext();) {
Path p = it.next();
Path rel = examplesDir.toPath().relativize(p);
String iri = version.getResource().getURI() + "example/"
+ rel.toString().substring(0, rel.toString().length() - 4);
String exampleTestSuiteName = version.toString() + " testing example " + rel;
testSuiteNames.put(iri, exampleTestSuiteName);
File exampleFile = p.toFile();
try {
new ReadExamples(exampleTestSuiteName).doJob(dataset, version, iri, exampleFile);
} catch (SAREFPipelineException ex) {
LOG.error(version.toString() + " Found errors for example " + rel
+ ". This example will be ignored.");
}
}
} catch(Exception ex) {
LOG.error(version.toString() + " Error while walking through the examples. They will be ignored.", ex);
continue;
}
}
}
private static void checkOWLProfile() {
dataset.begin(ReadWrite.READ);
Model config = dataset.getNamedModel(Constants.CONFIG);
for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.OntologyVersion); it.hasNext();) {
String ontology = it.next().getURI();
String testSuiteName = testSuiteNames.get(ontology);
try {
new CheckOWLProfile(testSuiteName).doJob(dataset, ontology);
} catch (SAREFPipelineException ex) {
LOG.warn("Found errors for ontology " + ontology, ex);
}
}
for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.Example); it.hasNext();) {
String ontology = it.next().getURI();
String testSuiteName = testSuiteNames.get(ontology);
try {
new CheckOWLProfile(testSuiteName).doJob(dataset, ontology);
} catch (SAREFPipelineException ex) {
LOG.warn("Found errors for example " + ontology, ex);
}
}
dataset.end();
}
private static SPARQLExtStreamManager initializeStreamManager() throws IOException {
File documentationDir = new File(Main.class.getClassLoader().getResource("documentation").getFile());
Path dirPath = Paths.get(documentationDir.toURI());
LocatorFileAccept locator = new LocatorFileAccept(documentationDir.toURI().getPath());
LocationMapperAccept mapper = new LocationMapperAccept();
SPARQLExtStreamManager sm = SPARQLExtStreamManager.makeStreamManager(locator);
sm.setLocationMapper(mapper);
Files.walk(dirPath).filter((p) -> {
return p.toFile().isFile();
}).forEach((p) -> {
String relativePath = dirPath.relativize(p).toString();
String fileurl = Constants.BASE_DOC + relativePath.replace("\\", "/");
mapper.addAltEntry(fileurl, p.toString());
});
return sm;
}
private static void resetCheckout(Repository repository) {
try(Git git = Git.open(repository.getDirectory())) {
RevCommit currentRevCommit = repository.getCurrentRevCommit();
if(currentRevCommit != null) {
git.checkout().setStartPoint(currentRevCommit).call();
}
} catch (IOException | GitAPIException ex) {
LOG.warn("Error while reseting repository " + repository, ex);
}
}
private static void reportAndExit(int code) {
try {
File report = new File(target, "report_output.xml");
JAXBContext jaxbContext = JAXBContext.newInstance(TestSuites.class);
Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
jaxbMarshaller.marshal(testSuites, report);
jaxbMarshaller.marshal(testSuites, System.out);
final StringWriter sw = new StringWriter();
jaxbMarshaller.marshal(testSuites, sw);
// generate the report.html
File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html");
try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) {
Context context = ContextUtils.build(writer).setBase(Constants.BASE)
.setDebugTemplate(!Constants.PRODUCTION).setStreamManager(streamManager).build();
BindingHashMap binding = new BindingHashMap();
binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString()));
List<Binding> bindings = new ArrayList<>();
bindings.add(binding);
String query = IOUtils.toString(
streamManager.open(new LookUpRequest("report/main.rqg", SPARQLExt.MEDIA_TYPE)),
StandardCharsets.UTF_8);
RootPlan reportPlan = PlanFactory.create(query, Constants.BASE_DOC);
reportPlan.execTemplateStream(bindings, context);
}
if (openBrowser) {
if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) {
Desktop.getDesktop().browse(new URI(
Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8")));
Desktop.getDesktop().browse(reportHTML.toURI());
// Desktop.getDesktop().browse(new URI(
// Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8")));
}
}
} catch (JAXBException | URISyntaxException | IOException ex) {
// } catch (JAXBException | URISyntaxException | IOException ex) {
} catch (JAXBException | IOException ex) {
LOG.error("Exception:", ex);
ex.printStackTrace();
}
System.exit(code);
}
private static <T> T doJob(JobRunner<T> checker) {
T result = checker.doJob(testSuites);
if (testSuites.getErrors() > 0) {
reportAndExit(-1);
}
return result;
System.exit(code);
}
}
package fr.emse.gitlab.saref;
public class SAREFPipelineException extends Exception {
private final String type;
public SAREFPipelineException() {
this.type = null;
}
public SAREFPipelineException(String type) {
this.type = type;
}
public SAREFPipelineException(String type, String message) {
super(message);
this.type = type;
}
public SAREFPipelineException(String type, String message, Throwable cause) {
super(message, cause);
this.type = type;
}
public SAREFPipelineException(String type, Throwable cause) {
super(cause);
this.type = type;
}
public String getType() {
return type;
}
}
......@@ -8,8 +8,8 @@ public class BranchVersion extends Version {
final String branchName;
public BranchVersion(String name, Ref ref, Date issued, String branchName) {
super(name, ref, issued);
BranchVersion(Repository repository, Ref ref, Date issued, String branchName) {
super(repository, ref, issued);
this.branchName = branchName;
}
......@@ -24,12 +24,7 @@ public class BranchVersion extends Version {
}
@Override
public String toString() {
return String.format("Version[%s, %s])", name, branchName);
}
@Override
public String getVersionInfo() {
public String getVersionName() {
return branchName;
}
}
......@@ -9,8 +9,8 @@ public class MasterVersion extends Version {
private static final String VERSION_INFO = "master";
public MasterVersion(String name, Ref ref, Date issued) {
super(name, ref, issued);
MasterVersion(Repository repository, Ref ref, Date issued) {
super(repository, ref, issued);
}
@Override
......@@ -22,14 +22,9 @@ public class MasterVersion extends Version {
public String getUri() {
return String.format("%s%s/", super.getSuperUri(), VERSION_INFO);
}
@Override
public String toString() {
return String.format("Version[%s, %s])", name, VERSION_INFO);
}
@Override
public String getVersionInfo() {
public String getVersionName() {
return VERSION_INFO;
}