Loading pom.xml +6 −0 Original line number Diff line number Diff line Loading @@ -138,6 +138,11 @@ <artifactId>jena-tdb</artifactId> <version>3.13.0</version> </dependency> <dependency> <groupId>fr.emse.ci</groupId> <artifactId>sparql-generate-xml</artifactId> <version>${sparql-generate.version}</version> </dependency> <dependency> <groupId>fr.emse.ci</groupId> <artifactId>sparql-generate-markdown</artifactId> Loading Loading @@ -182,6 +187,7 @@ </goals> <configuration> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>fr.emse.gitlab.saref.Main Loading src/main/java/fr/emse/gitlab/saref/CMDConfigurations.java +9 −5 Original line number Diff line number Diff line Loading @@ -27,13 +27,16 @@ public class CMDConfigurations { public static final String ARG_DIRECTORY_MAN = "Location of the SAREF extension directory (default is .)"; public static final String ARG_INCLUDE_ALL = "a"; public static final String ARG_INCLUDE_ALL_LONG = "all"; public static final String ARG_INCLUDE_ALL_MAN = "Include all branches (not for production)"; public static final String ARG_INCLUDE_ALL_MAN = "Include all branches"; public static final String ARG_INCLUDE_MASTER = "m"; public static final String ARG_INCLUDE_MASTER_LONG = "master"; public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches (not for production)"; public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches"; public static final String ARG_PRODUCTION = "p"; public static final String ARG_PRODUCTION_LONG = "prod"; public static final String ARG_PRODUCTION_MAN = "Production mode"; public static final String ARG_PRODUCTION_LONG = "production"; public static final String ARG_PRODUCTION_MAN = "Production mode: the directory does not contain a SAREF extension, but only a file `.saref-repositories.yml`"; public static final String ARG_DEBUG_TEMPLATE = "debug"; public static final String ARG_DEBUG_TEMPLATE_LONG = "debug-template"; public static final String ARG_DEBUG_TEMPLATE_MAN = "Use the debug-template mode of SPARQL-Generate when generating the documentation"; public static CommandLine parseArguments(String[] args) throws ParseException { DefaultParser commandLineParser = new DefaultParser(); Loading @@ -47,7 +50,8 @@ public class CMDConfigurations { .addOption(ARG_DIRECTORY, ARG_DIRECTORY_LONG, true, ARG_DIRECTORY_MAN) .addOption(ARG_INCLUDE_MASTER, ARG_INCLUDE_MASTER_LONG, false, ARG_INCLUDE_MASTER_MAN) .addOption(ARG_INCLUDE_ALL, ARG_INCLUDE_ALL_LONG, false, ARG_INCLUDE_ALL_MAN) .addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN); .addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN) .addOption(ARG_DEBUG_TEMPLATE, ARG_DEBUG_TEMPLATE_LONG, false, ARG_DEBUG_TEMPLATE_MAN); } public static void displayHelp() { Loading src/main/java/fr/emse/gitlab/saref/Constants.java +12 −2 Original line number Diff line number Diff line package fr.emse.gitlab.saref; import java.io.File; import java.nio.file.FileSystems; import java.nio.file.PathMatcher; import java.util.regex.Pattern; public class Constants { public final static String BASE = "https://saref.etsi.org/"; public final static String BASE_DOC = BASE + "documentation/"; public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger"; public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})"; public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)"; public static final String REGEX_MASTER_BRANCH = "^refs/remotes/origin/master$"; public static final String MASTER_BRANCH = "refs/remotes/origin/master"; public static final String REGEX_RELEASE_BRANCH = "^refs/remotes/origin/release-" + REGEX_VERSION + "$"; public static final String REGEX_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$"; public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH); Loading @@ -27,7 +32,6 @@ public class Constants { // output public static final String TARGET_DIR = "target"; public static final String GIT_DIR = TARGET_DIR + File.separator + "sources"; public static final String DATASET_DIR = TARGET_DIR + File.separator + "tdb"; public static final String LOG_FILE_NAME = TARGET_DIR + File.separator + "output.log"; public static final String SITE_DIR = TARGET_DIR + File.separator + "site"; Loading @@ -38,8 +42,14 @@ public class Constants { public static final String propertyFile = ".saref-repositories"; public static final String CONFIG = BASE + "config"; public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl"); public static boolean INCLUDE_MASTER = false; public static boolean INCLUDE_ALL = false; public static boolean PRODUCTION = false; public static boolean DEBUG_TEMPLATE = false; } src/main/java/fr/emse/gitlab/saref/Main.java +261 −55 Original line number Diff line number Diff line package fr.emse.gitlab.saref; import static fr.emse.gitlab.saref.CMDConfigurations.*; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DEBUG_TEMPLATE; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_ALL; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_PRODUCTION; import java.awt.Desktop; import java.io.File; import java.io.FileReader; import java.io.FileOutputStream; import java.io.IOException; import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Map; import java.util.function.Consumer; import javax.net.ssl.HttpsURLConnection; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.jena.atlas.io.IndentedWriter; import org.apache.jena.graph.NodeFactory; import org.apache.jena.query.Dataset; import org.apache.jena.query.ReadWrite; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.binding.Binding; import org.apache.jena.sparql.engine.binding.BindingHashMap; import org.apache.jena.sparql.util.Context; import org.apache.jena.tdb.TDBFactory; import org.apache.jena.vocabulary.RDF; import org.apache.log4j.Layout; import org.apache.log4j.PatternLayout; import org.semanticweb.owlapi.model.OWLOntologyIRIMapper; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.CheckoutConflictException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.revwalk.RevCommit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import fr.emse.gitlab.saref.entities.git.Repository; import fr.emse.ci.sparqlext.SPARQLExt; import fr.emse.ci.sparqlext.engine.PlanFactory; import fr.emse.ci.sparqlext.engine.RootPlan; import fr.emse.ci.sparqlext.stream.LocationMapperAccept; import fr.emse.ci.sparqlext.stream.LocatorFileAccept; import fr.emse.ci.sparqlext.stream.LookUpRequest; import fr.emse.ci.sparqlext.stream.SPARQLExtStreamManager; import fr.emse.ci.sparqlext.utils.ContextUtils; import fr.emse.ci.sparqlext.utils.VarUtils; import fr.emse.gitlab.saref.entities.git.Repositories; import fr.emse.gitlab.saref.entities.git.Repository; import fr.emse.gitlab.saref.entities.git.Version; import fr.emse.gitlab.saref.entities.tests.TestSuites; import fr.emse.gitlab.saref.jobs.CreateDataset; import fr.emse.gitlab.saref.jobs.JobRunner; import fr.emse.gitlab.saref.jobs.ReadRepositories; import fr.emse.gitlab.saref.jobs.WriteRDFFiles; import fr.emse.gitlab.saref.jobs.CheckConfig; import fr.emse.gitlab.saref.jobs.CheckOWLProfile; import fr.emse.gitlab.saref.jobs.CheckRepositoryStructure; import fr.emse.gitlab.saref.jobs.CheckShapes; import fr.emse.gitlab.saref.jobs.CopyStaticFiles; import fr.emse.gitlab.saref.jobs.CheckoutJob; import fr.emse.gitlab.saref.jobs.ReadExamples; import fr.emse.gitlab.saref.jobs.ReadOntology; import fr.emse.gitlab.saref.jobs.ReadRepositories; import fr.emse.gitlab.saref.jobs.GeneratePortal; import fr.emse.gitlab.saref.utils.TestSuitesAppender; import fr.emse.gitlab.saref.vocabs.EX; public class Main { static final Logger LOG = LoggerFactory.getLogger(Main.class); private static final Layout LAYOUT = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n"); private static final org.apache.log4j.Logger ROOT_LOGGER = org.apache.log4j.Logger.getRootLogger(); static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_BASE + ".Main"); private static TestSuites testSuites = new TestSuites("SAREF pipeline"); private static final Map<String, String> testSuiteNames = new HashMap<>(); private static TestSuites testSuites = new TestSuites(); private static File directory; private static File target; private static boolean openBrowser = false; private static SPARQLExtStreamManager streamManager; private static boolean openBrowser = true; private static Dataset dataset; private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites"); public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException { throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException { parseCommandArguments(args); setLogAppenders(); prepareDirectory(); streamManager = initializeStreamManager(); dataset = createFreshDataset(); LOG.info("Starting pipeline in " + directory); Repositories repositories = readRepositories(); accept(repositories, Main::testRepository); checkOWLProfile(); new CheckConfig().doJob(dataset); new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory); accept(repositories, Main::resetCheckout); testSuites.prune(); reportAndExit(0); } private static void parseCommandArguments(String[] args) throws ParseException, IOException { CommandLine cl = CMDConfigurations.parseArguments(args); if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) { CMDConfigurations.displayHelp(); Loading @@ -73,67 +130,216 @@ public class Main { Constants.INCLUDE_ALL = true; } if (cl.hasOption(ARG_PRODUCTION)) { if(cl.hasOption(ARG_INCLUDE_MASTER) || cl.hasOption(ARG_INCLUDE_ALL)) { throw new RuntimeException("Option --production is incompatible with options --master or --all "); } Constants.PRODUCTION = true; } if (cl.hasOption(ARG_DEBUG_TEMPLATE)) { Constants.DEBUG_TEMPLATE = true; } String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT); if (dirName.equals("")) { dirName = ARG_DIRECTORY_DEFAULT; } directory = new File(dirName).getCanonicalFile(); } directory = new File(dirName).getAbsoluteFile(); private static void setLogAppenders() throws IOException { File logFile = new File(directory, Constants.LOG_FILE_NAME); Layout layout = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n"); org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger(); rootLogger.addAppender(new org.apache.log4j.RollingFileAppender(layout, logFile.getAbsolutePath(), false)); org.apache.log4j.Logger loggerBase = org.apache.log4j.Logger.getLogger(Constants.LOGGER_BASE); TestSuitesAppender appender = new TestSuitesAppender(testSuites); loggerBase.addAppender(appender); } private static void prepareDirectory() throws IOException { target = new File(directory, Constants.TARGET_DIR); FileUtils.forceMkdir(target); File siteDir = new File(directory, Constants.SITE_DIR); File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR); FileUtils.forceMkdir(siteDir); FileUtils.forceMkdir(staticTargetDir); File staticDir = new File(Main.class.getClassLoader().getResource("static").getFile()); FileUtils.copyDirectory(staticDir, staticTargetDir); } File logFile = new File(directory, Constants.LOG_FILE_NAME); ROOT_LOGGER.addAppender(new org.apache.log4j.RollingFileAppender(LAYOUT, logFile.getAbsolutePath(), false)); LOG.info("Starting pipeline"); doJob(new CheckRepositoryStructure(directory)); // doJob(new CheckShapes(directory)); // Repositories repositories = doJob(new ReadRepositories(directory)); // Repositories repositories = new Repositories(); // Dataset dataset = doJob(new CreateDataset(directory, repositories)); // doJob(new CopyStaticFiles(directory)); // Set<OWLOntologyIRIMapper> mappers = doJob(new WriteRDFFiles(directory, dataset)); // doJob(new CheckOWLProfile(directory, mappers)); reportAndExit(0); private static Dataset createFreshDataset() throws IOException { FileUtils.forceMkdir(new File(directory, Constants.DATASET_DIR)); dataset = TDBFactory.createDataset(Constants.DATASET_DIR); dataset.begin(ReadWrite.WRITE); dataset.getDefaultModel().removeAll(); for (Iterator<String> it = dataset.listNames(); it.hasNext();) { String name = it.next(); dataset.removeNamedModel(name); } dataset.addNamedModel(Constants.CONFIG, ModelFactory.createDefaultModel()); dataset.commit(); return dataset; } private static void accept(Repositories repositories, Consumer<Repository> consumer) { if (repositories.getDefaultRepository() != null) { Repository repository = repositories.getDefaultRepository(); consumer.accept(repository); } for(Repository repository : repositories.getNamedRepositories()) { consumer.accept(repository); } } private static Repositories readRepositories() { try { return new ReadRepositories("Fetching required repositories").readRepositories(directory); } catch (Exception ex) { reportAndExit(-1); return null; } } private static void testRepository(Repository repository) { for (Version version : repository.getVersions()) { try { String versionTestSuiteName = version.toString() + " - testing repository structure"; new CheckoutJob(versionTestSuiteName).checkoutVersion(version); new CheckRepositoryStructure(versionTestSuiteName).check(version); } catch(SAREFPipelineException ex) { LOG.error("Error while testing repository structure " + version, ex); continue; } try { String ontologyTestSuiteName = version.toString() + " - testing ontology file"; testSuiteNames.put(version.getUri(), ontologyTestSuiteName); new ReadOntology(ontologyTestSuiteName).doJob(dataset, version); } catch(SAREFPipelineException ex) { LOG.error(version.toString() + " Found errors for " + version.toString() + ". This version and all the examples will be ignored.", ex); continue; } File examplesDir = new File(version.getRepository().getDirectory(), "examples"); try { for (Iterator<Path> it = Files.walk(examplesDir.toPath()).filter(p -> { return Constants.ttlMatcher.matches(p); }).iterator();it.hasNext();) { Path p = it.next(); Path rel = examplesDir.toPath().relativize(p); String iri = version.getResource().getURI() + "example/" + rel.toString().substring(0, rel.toString().length() - 4); String exampleTestSuiteName = version.toString() + " testing example " + rel; testSuiteNames.put(iri, exampleTestSuiteName); File exampleFile = p.toFile(); try { new ReadExamples(exampleTestSuiteName).doJob(dataset, version, iri, exampleFile); } catch (SAREFPipelineException ex) { LOG.error(version.toString() + " Found errors for example " + rel + ". This example will be ignored."); } } } catch(Exception ex) { LOG.error(version.toString() + " Error while walking through the examples. They will be ignored.", ex); continue; } } } private static void checkOWLProfile() { dataset.begin(ReadWrite.READ); Model config = dataset.getNamedModel(Constants.CONFIG); for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.OntologyVersion); it.hasNext();) { String ontology = it.next().getURI(); String testSuiteName = testSuiteNames.get(ontology); try { new CheckOWLProfile(testSuiteName).doJob(dataset, ontology); } catch (SAREFPipelineException ex) { LOG.warn("Found errors for ontology " + ontology, ex); } } for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.Example); it.hasNext();) { String ontology = it.next().getURI(); String testSuiteName = testSuiteNames.get(ontology); try { new CheckOWLProfile(testSuiteName).doJob(dataset, ontology); } catch (SAREFPipelineException ex) { LOG.warn("Found errors for example " + ontology, ex); } } dataset.end(); } private static SPARQLExtStreamManager initializeStreamManager() throws IOException { File documentationDir = new File(Main.class.getClassLoader().getResource("documentation").getFile()); Path dirPath = Paths.get(documentationDir.toURI()); LocatorFileAccept locator = new LocatorFileAccept(documentationDir.toURI().getPath()); LocationMapperAccept mapper = new LocationMapperAccept(); SPARQLExtStreamManager sm = SPARQLExtStreamManager.makeStreamManager(locator); sm.setLocationMapper(mapper); Files.walk(dirPath).filter((p) -> { return p.toFile().isFile(); }).forEach((p) -> { String relativePath = dirPath.relativize(p).toString(); String fileurl = Constants.BASE_DOC + relativePath.replace("\\", "/"); mapper.addAltEntry(fileurl, p.toString()); }); return sm; } private static void resetCheckout(Repository repository) { try(Git git = Git.open(repository.getDirectory())) { RevCommit currentRevCommit = repository.getCurrentRevCommit(); if(currentRevCommit != null) { git.checkout().setStartPoint(currentRevCommit).call(); } } catch (IOException | GitAPIException ex) { LOG.warn("Error while reseting repository " + repository, ex); } } private static void reportAndExit(int code) { try { File report = new File(target, "report_output.xml"); JAXBContext jaxbContext = JAXBContext.newInstance(TestSuites.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); jaxbMarshaller.marshal(testSuites, report); jaxbMarshaller.marshal(testSuites, System.out); final StringWriter sw = new StringWriter(); jaxbMarshaller.marshal(testSuites, sw); // generate the report.html File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html"); try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) { Context context = ContextUtils.build(writer).setBase(Constants.BASE) .setDebugTemplate(!Constants.PRODUCTION).setStreamManager(streamManager).build(); BindingHashMap binding = new BindingHashMap(); binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString())); List<Binding> bindings = new ArrayList<>(); bindings.add(binding); String query = IOUtils.toString( streamManager.open(new LookUpRequest("report/main.rqg", SPARQLExt.MEDIA_TYPE)), StandardCharsets.UTF_8); RootPlan reportPlan = PlanFactory.create(query, Constants.BASE_DOC); reportPlan.execTemplateStream(bindings, context); } if (openBrowser) { if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) { Desktop.getDesktop().browse(new URI( Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8"))); Desktop.getDesktop().browse(reportHTML.toURI()); // Desktop.getDesktop().browse(new URI( // Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8"))); } } } catch (JAXBException | URISyntaxException | IOException ex) { // } catch (JAXBException | URISyntaxException | IOException ex) { } catch (JAXBException | IOException ex) { LOG.error("Exception:", ex); ex.printStackTrace(); } System.exit(code); } private static <T> T doJob(JobRunner<T> checker) { T result = checker.doJob(testSuites); if (testSuites.getErrors() > 0) { reportAndExit(-1); } return result; System.exit(code); } } src/main/java/fr/emse/gitlab/saref/SAREFPipelineException.java 0 → 100644 +34 −0 Original line number Diff line number Diff line package fr.emse.gitlab.saref; public class SAREFPipelineException extends Exception { private final String type; public SAREFPipelineException() { this.type = null; } public SAREFPipelineException(String type) { this.type = type; } public SAREFPipelineException(String type, String message) { super(message); this.type = type; } public SAREFPipelineException(String type, String message, Throwable cause) { super(message, cause); this.type = type; } public SAREFPipelineException(String type, Throwable cause) { super(cause); this.type = type; } public String getType() { return type; } } Loading
pom.xml +6 −0 Original line number Diff line number Diff line Loading @@ -138,6 +138,11 @@ <artifactId>jena-tdb</artifactId> <version>3.13.0</version> </dependency> <dependency> <groupId>fr.emse.ci</groupId> <artifactId>sparql-generate-xml</artifactId> <version>${sparql-generate.version}</version> </dependency> <dependency> <groupId>fr.emse.ci</groupId> <artifactId>sparql-generate-markdown</artifactId> Loading Loading @@ -182,6 +187,7 @@ </goals> <configuration> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass>fr.emse.gitlab.saref.Main Loading
src/main/java/fr/emse/gitlab/saref/CMDConfigurations.java +9 −5 Original line number Diff line number Diff line Loading @@ -27,13 +27,16 @@ public class CMDConfigurations { public static final String ARG_DIRECTORY_MAN = "Location of the SAREF extension directory (default is .)"; public static final String ARG_INCLUDE_ALL = "a"; public static final String ARG_INCLUDE_ALL_LONG = "all"; public static final String ARG_INCLUDE_ALL_MAN = "Include all branches (not for production)"; public static final String ARG_INCLUDE_ALL_MAN = "Include all branches"; public static final String ARG_INCLUDE_MASTER = "m"; public static final String ARG_INCLUDE_MASTER_LONG = "master"; public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches (not for production)"; public static final String ARG_INCLUDE_MASTER_MAN = "Include the HEAD of the master branches"; public static final String ARG_PRODUCTION = "p"; public static final String ARG_PRODUCTION_LONG = "prod"; public static final String ARG_PRODUCTION_MAN = "Production mode"; public static final String ARG_PRODUCTION_LONG = "production"; public static final String ARG_PRODUCTION_MAN = "Production mode: the directory does not contain a SAREF extension, but only a file `.saref-repositories.yml`"; public static final String ARG_DEBUG_TEMPLATE = "debug"; public static final String ARG_DEBUG_TEMPLATE_LONG = "debug-template"; public static final String ARG_DEBUG_TEMPLATE_MAN = "Use the debug-template mode of SPARQL-Generate when generating the documentation"; public static CommandLine parseArguments(String[] args) throws ParseException { DefaultParser commandLineParser = new DefaultParser(); Loading @@ -47,7 +50,8 @@ public class CMDConfigurations { .addOption(ARG_DIRECTORY, ARG_DIRECTORY_LONG, true, ARG_DIRECTORY_MAN) .addOption(ARG_INCLUDE_MASTER, ARG_INCLUDE_MASTER_LONG, false, ARG_INCLUDE_MASTER_MAN) .addOption(ARG_INCLUDE_ALL, ARG_INCLUDE_ALL_LONG, false, ARG_INCLUDE_ALL_MAN) .addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN); .addOption(ARG_PRODUCTION, ARG_PRODUCTION_LONG, false, ARG_PRODUCTION_MAN) .addOption(ARG_DEBUG_TEMPLATE, ARG_DEBUG_TEMPLATE_LONG, false, ARG_DEBUG_TEMPLATE_MAN); } public static void displayHelp() { Loading
src/main/java/fr/emse/gitlab/saref/Constants.java +12 −2 Original line number Diff line number Diff line package fr.emse.gitlab.saref; import java.io.File; import java.nio.file.FileSystems; import java.nio.file.PathMatcher; import java.util.regex.Pattern; public class Constants { public final static String BASE = "https://saref.etsi.org/"; public final static String BASE_DOC = BASE + "documentation/"; public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger"; public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})"; public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)"; public static final String REGEX_MASTER_BRANCH = "^refs/remotes/origin/master$"; public static final String MASTER_BRANCH = "refs/remotes/origin/master"; public static final String REGEX_RELEASE_BRANCH = "^refs/remotes/origin/release-" + REGEX_VERSION + "$"; public static final String REGEX_OTHER_BRANCH = "^refs/remotes/origin/(?<name>[^/]+)$"; public static final Pattern REGEX_RELEASE_BRANCH_PATTERN = Pattern.compile(REGEX_RELEASE_BRANCH); Loading @@ -27,7 +32,6 @@ public class Constants { // output public static final String TARGET_DIR = "target"; public static final String GIT_DIR = TARGET_DIR + File.separator + "sources"; public static final String DATASET_DIR = TARGET_DIR + File.separator + "tdb"; public static final String LOG_FILE_NAME = TARGET_DIR + File.separator + "output.log"; public static final String SITE_DIR = TARGET_DIR + File.separator + "site"; Loading @@ -38,8 +42,14 @@ public class Constants { public static final String propertyFile = ".saref-repositories"; public static final String CONFIG = BASE + "config"; public static final PathMatcher ttlMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.ttl"); public static boolean INCLUDE_MASTER = false; public static boolean INCLUDE_ALL = false; public static boolean PRODUCTION = false; public static boolean DEBUG_TEMPLATE = false; }
src/main/java/fr/emse/gitlab/saref/Main.java +261 −55 Original line number Diff line number Diff line package fr.emse.gitlab.saref; import static fr.emse.gitlab.saref.CMDConfigurations.*; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DEBUG_TEMPLATE; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_ALL; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER; import static fr.emse.gitlab.saref.CMDConfigurations.ARG_PRODUCTION; import java.awt.Desktop; import java.io.File; import java.io.FileReader; import java.io.FileOutputStream; import java.io.IOException; import java.io.StringWriter; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.Set; import java.util.Map; import java.util.function.Consumer; import javax.net.ssl.HttpsURLConnection; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.jena.atlas.io.IndentedWriter; import org.apache.jena.graph.NodeFactory; import org.apache.jena.query.Dataset; import org.apache.jena.query.ReadWrite; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; import org.apache.jena.rdf.model.ResIterator; import org.apache.jena.sparql.core.Var; import org.apache.jena.sparql.engine.binding.Binding; import org.apache.jena.sparql.engine.binding.BindingHashMap; import org.apache.jena.sparql.util.Context; import org.apache.jena.tdb.TDBFactory; import org.apache.jena.vocabulary.RDF; import org.apache.log4j.Layout; import org.apache.log4j.PatternLayout; import org.semanticweb.owlapi.model.OWLOntologyIRIMapper; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.CheckoutConflictException; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.revwalk.RevCommit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import fr.emse.gitlab.saref.entities.git.Repository; import fr.emse.ci.sparqlext.SPARQLExt; import fr.emse.ci.sparqlext.engine.PlanFactory; import fr.emse.ci.sparqlext.engine.RootPlan; import fr.emse.ci.sparqlext.stream.LocationMapperAccept; import fr.emse.ci.sparqlext.stream.LocatorFileAccept; import fr.emse.ci.sparqlext.stream.LookUpRequest; import fr.emse.ci.sparqlext.stream.SPARQLExtStreamManager; import fr.emse.ci.sparqlext.utils.ContextUtils; import fr.emse.ci.sparqlext.utils.VarUtils; import fr.emse.gitlab.saref.entities.git.Repositories; import fr.emse.gitlab.saref.entities.git.Repository; import fr.emse.gitlab.saref.entities.git.Version; import fr.emse.gitlab.saref.entities.tests.TestSuites; import fr.emse.gitlab.saref.jobs.CreateDataset; import fr.emse.gitlab.saref.jobs.JobRunner; import fr.emse.gitlab.saref.jobs.ReadRepositories; import fr.emse.gitlab.saref.jobs.WriteRDFFiles; import fr.emse.gitlab.saref.jobs.CheckConfig; import fr.emse.gitlab.saref.jobs.CheckOWLProfile; import fr.emse.gitlab.saref.jobs.CheckRepositoryStructure; import fr.emse.gitlab.saref.jobs.CheckShapes; import fr.emse.gitlab.saref.jobs.CopyStaticFiles; import fr.emse.gitlab.saref.jobs.CheckoutJob; import fr.emse.gitlab.saref.jobs.ReadExamples; import fr.emse.gitlab.saref.jobs.ReadOntology; import fr.emse.gitlab.saref.jobs.ReadRepositories; import fr.emse.gitlab.saref.jobs.GeneratePortal; import fr.emse.gitlab.saref.utils.TestSuitesAppender; import fr.emse.gitlab.saref.vocabs.EX; public class Main { static final Logger LOG = LoggerFactory.getLogger(Main.class); private static final Layout LAYOUT = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n"); private static final org.apache.log4j.Logger ROOT_LOGGER = org.apache.log4j.Logger.getRootLogger(); static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_BASE + ".Main"); private static TestSuites testSuites = new TestSuites("SAREF pipeline"); private static final Map<String, String> testSuiteNames = new HashMap<>(); private static TestSuites testSuites = new TestSuites(); private static File directory; private static File target; private static boolean openBrowser = false; private static SPARQLExtStreamManager streamManager; private static boolean openBrowser = true; private static Dataset dataset; private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites"); public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException { throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException { parseCommandArguments(args); setLogAppenders(); prepareDirectory(); streamManager = initializeStreamManager(); dataset = createFreshDataset(); LOG.info("Starting pipeline in " + directory); Repositories repositories = readRepositories(); accept(repositories, Main::testRepository); checkOWLProfile(); new CheckConfig().doJob(dataset); new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory); accept(repositories, Main::resetCheckout); testSuites.prune(); reportAndExit(0); } private static void parseCommandArguments(String[] args) throws ParseException, IOException { CommandLine cl = CMDConfigurations.parseArguments(args); if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) { CMDConfigurations.displayHelp(); Loading @@ -73,67 +130,216 @@ public class Main { Constants.INCLUDE_ALL = true; } if (cl.hasOption(ARG_PRODUCTION)) { if(cl.hasOption(ARG_INCLUDE_MASTER) || cl.hasOption(ARG_INCLUDE_ALL)) { throw new RuntimeException("Option --production is incompatible with options --master or --all "); } Constants.PRODUCTION = true; } if (cl.hasOption(ARG_DEBUG_TEMPLATE)) { Constants.DEBUG_TEMPLATE = true; } String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT); if (dirName.equals("")) { dirName = ARG_DIRECTORY_DEFAULT; } directory = new File(dirName).getCanonicalFile(); } directory = new File(dirName).getAbsoluteFile(); private static void setLogAppenders() throws IOException { File logFile = new File(directory, Constants.LOG_FILE_NAME); Layout layout = new PatternLayout("%d{mm:ss,SSS} %t %-5p %c:%L - %m%n"); org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger(); rootLogger.addAppender(new org.apache.log4j.RollingFileAppender(layout, logFile.getAbsolutePath(), false)); org.apache.log4j.Logger loggerBase = org.apache.log4j.Logger.getLogger(Constants.LOGGER_BASE); TestSuitesAppender appender = new TestSuitesAppender(testSuites); loggerBase.addAppender(appender); } private static void prepareDirectory() throws IOException { target = new File(directory, Constants.TARGET_DIR); FileUtils.forceMkdir(target); File siteDir = new File(directory, Constants.SITE_DIR); File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR); FileUtils.forceMkdir(siteDir); FileUtils.forceMkdir(staticTargetDir); File staticDir = new File(Main.class.getClassLoader().getResource("static").getFile()); FileUtils.copyDirectory(staticDir, staticTargetDir); } File logFile = new File(directory, Constants.LOG_FILE_NAME); ROOT_LOGGER.addAppender(new org.apache.log4j.RollingFileAppender(LAYOUT, logFile.getAbsolutePath(), false)); LOG.info("Starting pipeline"); doJob(new CheckRepositoryStructure(directory)); // doJob(new CheckShapes(directory)); // Repositories repositories = doJob(new ReadRepositories(directory)); // Repositories repositories = new Repositories(); // Dataset dataset = doJob(new CreateDataset(directory, repositories)); // doJob(new CopyStaticFiles(directory)); // Set<OWLOntologyIRIMapper> mappers = doJob(new WriteRDFFiles(directory, dataset)); // doJob(new CheckOWLProfile(directory, mappers)); reportAndExit(0); private static Dataset createFreshDataset() throws IOException { FileUtils.forceMkdir(new File(directory, Constants.DATASET_DIR)); dataset = TDBFactory.createDataset(Constants.DATASET_DIR); dataset.begin(ReadWrite.WRITE); dataset.getDefaultModel().removeAll(); for (Iterator<String> it = dataset.listNames(); it.hasNext();) { String name = it.next(); dataset.removeNamedModel(name); } dataset.addNamedModel(Constants.CONFIG, ModelFactory.createDefaultModel()); dataset.commit(); return dataset; } private static void accept(Repositories repositories, Consumer<Repository> consumer) { if (repositories.getDefaultRepository() != null) { Repository repository = repositories.getDefaultRepository(); consumer.accept(repository); } for(Repository repository : repositories.getNamedRepositories()) { consumer.accept(repository); } } private static Repositories readRepositories() { try { return new ReadRepositories("Fetching required repositories").readRepositories(directory); } catch (Exception ex) { reportAndExit(-1); return null; } } private static void testRepository(Repository repository) { for (Version version : repository.getVersions()) { try { String versionTestSuiteName = version.toString() + " - testing repository structure"; new CheckoutJob(versionTestSuiteName).checkoutVersion(version); new CheckRepositoryStructure(versionTestSuiteName).check(version); } catch(SAREFPipelineException ex) { LOG.error("Error while testing repository structure " + version, ex); continue; } try { String ontologyTestSuiteName = version.toString() + " - testing ontology file"; testSuiteNames.put(version.getUri(), ontologyTestSuiteName); new ReadOntology(ontologyTestSuiteName).doJob(dataset, version); } catch(SAREFPipelineException ex) { LOG.error(version.toString() + " Found errors for " + version.toString() + ". This version and all the examples will be ignored.", ex); continue; } File examplesDir = new File(version.getRepository().getDirectory(), "examples"); try { for (Iterator<Path> it = Files.walk(examplesDir.toPath()).filter(p -> { return Constants.ttlMatcher.matches(p); }).iterator();it.hasNext();) { Path p = it.next(); Path rel = examplesDir.toPath().relativize(p); String iri = version.getResource().getURI() + "example/" + rel.toString().substring(0, rel.toString().length() - 4); String exampleTestSuiteName = version.toString() + " testing example " + rel; testSuiteNames.put(iri, exampleTestSuiteName); File exampleFile = p.toFile(); try { new ReadExamples(exampleTestSuiteName).doJob(dataset, version, iri, exampleFile); } catch (SAREFPipelineException ex) { LOG.error(version.toString() + " Found errors for example " + rel + ". This example will be ignored."); } } } catch(Exception ex) { LOG.error(version.toString() + " Error while walking through the examples. They will be ignored.", ex); continue; } } } private static void checkOWLProfile() { dataset.begin(ReadWrite.READ); Model config = dataset.getNamedModel(Constants.CONFIG); for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.OntologyVersion); it.hasNext();) { String ontology = it.next().getURI(); String testSuiteName = testSuiteNames.get(ontology); try { new CheckOWLProfile(testSuiteName).doJob(dataset, ontology); } catch (SAREFPipelineException ex) { LOG.warn("Found errors for ontology " + ontology, ex); } } for (ResIterator it = config.listResourcesWithProperty(RDF.type, EX.Example); it.hasNext();) { String ontology = it.next().getURI(); String testSuiteName = testSuiteNames.get(ontology); try { new CheckOWLProfile(testSuiteName).doJob(dataset, ontology); } catch (SAREFPipelineException ex) { LOG.warn("Found errors for example " + ontology, ex); } } dataset.end(); } private static SPARQLExtStreamManager initializeStreamManager() throws IOException { File documentationDir = new File(Main.class.getClassLoader().getResource("documentation").getFile()); Path dirPath = Paths.get(documentationDir.toURI()); LocatorFileAccept locator = new LocatorFileAccept(documentationDir.toURI().getPath()); LocationMapperAccept mapper = new LocationMapperAccept(); SPARQLExtStreamManager sm = SPARQLExtStreamManager.makeStreamManager(locator); sm.setLocationMapper(mapper); Files.walk(dirPath).filter((p) -> { return p.toFile().isFile(); }).forEach((p) -> { String relativePath = dirPath.relativize(p).toString(); String fileurl = Constants.BASE_DOC + relativePath.replace("\\", "/"); mapper.addAltEntry(fileurl, p.toString()); }); return sm; } private static void resetCheckout(Repository repository) { try(Git git = Git.open(repository.getDirectory())) { RevCommit currentRevCommit = repository.getCurrentRevCommit(); if(currentRevCommit != null) { git.checkout().setStartPoint(currentRevCommit).call(); } } catch (IOException | GitAPIException ex) { LOG.warn("Error while reseting repository " + repository, ex); } } private static void reportAndExit(int code) { try { File report = new File(target, "report_output.xml"); JAXBContext jaxbContext = JAXBContext.newInstance(TestSuites.class); Marshaller jaxbMarshaller = jaxbContext.createMarshaller(); jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); jaxbMarshaller.marshal(testSuites, report); jaxbMarshaller.marshal(testSuites, System.out); final StringWriter sw = new StringWriter(); jaxbMarshaller.marshal(testSuites, sw); // generate the report.html File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html"); try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) { Context context = ContextUtils.build(writer).setBase(Constants.BASE) .setDebugTemplate(!Constants.PRODUCTION).setStreamManager(streamManager).build(); BindingHashMap binding = new BindingHashMap(); binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString())); List<Binding> bindings = new ArrayList<>(); bindings.add(binding); String query = IOUtils.toString( streamManager.open(new LookUpRequest("report/main.rqg", SPARQLExt.MEDIA_TYPE)), StandardCharsets.UTF_8); RootPlan reportPlan = PlanFactory.create(query, Constants.BASE_DOC); reportPlan.execTemplateStream(bindings, context); } if (openBrowser) { if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) { Desktop.getDesktop().browse(new URI( Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8"))); Desktop.getDesktop().browse(reportHTML.toURI()); // Desktop.getDesktop().browse(new URI( // Constants.BASE + "report.html?report=" + URLEncoder.encode(sw.toString(), "UTF-8"))); } } } catch (JAXBException | URISyntaxException | IOException ex) { // } catch (JAXBException | URISyntaxException | IOException ex) { } catch (JAXBException | IOException ex) { LOG.error("Exception:", ex); ex.printStackTrace(); } System.exit(code); } private static <T> T doJob(JobRunner<T> checker) { T result = checker.doJob(testSuites); if (testSuites.getErrors() > 0) { reportAndExit(-1); } return result; System.exit(code); } }
src/main/java/fr/emse/gitlab/saref/SAREFPipelineException.java 0 → 100644 +34 −0 Original line number Diff line number Diff line package fr.emse.gitlab.saref; public class SAREFPipelineException extends Exception { private final String type; public SAREFPipelineException() { this.type = null; } public SAREFPipelineException(String type) { this.type = type; } public SAREFPipelineException(String type, String message) { super(message); this.type = type; } public SAREFPipelineException(String type, String message, Throwable cause) { super(message, cause); this.type = type; } public SAREFPipelineException(String type, Throwable cause) { super(cause); this.type = type; } public String getType() { return type; } }