Commit 24e19fb5 authored by Maxime Lefrançois's avatar Maxime Lefrançois
Browse files

pull repository saref-portal-static

parent 67ecd2b7
......@@ -175,27 +175,6 @@
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.2.0</version>
<configuration>
<finalName>static</finalName>
<appendAssemblyId>false</appendAssemblyId>
<descriptors>
<descriptor>src/main/static/assembly.xml</descriptor>
</descriptors>
<outputDirectory>${project.build.outputDirectory}</outputDirectory>
</configuration>
<executions>
<execution>
<id>static</id>
<phase>process-resources</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
......
......@@ -7,10 +7,14 @@ import java.util.regex.Pattern;
public class Constants {
public final static String BASE = "https://saref.etsi.org/";
public final static String BASE_DOC = BASE + "documentation/";
public final static String LOGGER_BASE = "fr.emse.gitlab.saref.logger";
public static final String REGEX_EXT = "(?<ext>core|saref4[a-z]{4})";
public final static String SAREF_PORTAL_STATIC_GIT = "https://gitlab.emse.fr/saref/saref-portal-static.git";
// public final static String SAREF_PORTAL_STATIC_GIT = "https://forge.etsi.org/rep/SAREF/saref-portal-static.git"; // uncomment when the project is public
public static final String REGEX_EXT = "(?<ext>core|saref4[a-z0-9]{4})";
public static final String REGEX_VERSION = "v(?<major>[1-9][0-9]*)\\.(?<minor>[0-9]+)\\.(?<patch>[0-9]+)";
public static final String REGEX_ONTO_SERIES_URI = "^" + BASE + REGEX_EXT + "/$";
......
package fr.emse.gitlab.saref;
import static fr.emse.gitlab.saref.CMDConfigurations.*;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_DIRECTORY_DEFAULT;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_HELP;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_IGNORE_EXAMPLES;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_IGNORE_GIT;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_IGNORE_TERMS;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_INCLUDE_MASTER;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_NO_SITE;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_REMOTE_ONLY;
import static fr.emse.gitlab.saref.CMDConfigurations.ARG_VERBOSE;
import java.awt.Desktop;
import java.io.File;
......@@ -9,7 +18,6 @@ import java.io.IOException;
import java.io.StringWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
......@@ -33,6 +41,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.jena.atlas.io.IndentedWriter;
import org.apache.jena.atlas.web.TypedInputStream;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.ReadWrite;
......@@ -97,7 +106,6 @@ public class Main {
private static Dataset dataset;
private static Var VAR_TEST_SUITES = VarUtils.allocVar("testsuites");
static boolean remoteOnly;
static boolean ignoreGit;
static boolean includeMaster;
......@@ -105,12 +113,11 @@ public class Main {
static boolean ignoreTerms;
static boolean ignoreExamples;
static boolean verbose;
public static void main(String[] args)
throws IOException, InterruptedException, URISyntaxException, JAXBException, ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException, CheckoutConflictException, GitAPIException {
public static void main(String[] args) throws IOException, InterruptedException, URISyntaxException, JAXBException,
ParseException, RefAlreadyExistsException, RefNotFoundException, InvalidRefNameException,
CheckoutConflictException, GitAPIException {
CommandLine cl = CMDConfigurations.parseArguments(args);
if (cl.getOptions().length == 0 || cl.hasOption(ARG_HELP)) {
CMDConfigurations.displayHelp();
......@@ -119,7 +126,7 @@ public class Main {
String dirName = cl.getOptionValue(ARG_DIRECTORY, ARG_DIRECTORY_DEFAULT);
directory = new File(dirName).getCanonicalFile();
if(!directory.isDirectory()) {
if (!directory.isDirectory()) {
LOG.error("The directory does not exist " + directory);
System.exit(-1);
}
......@@ -131,7 +138,7 @@ public class Main {
ignoreExamples = cl.hasOption(ARG_IGNORE_EXAMPLES);
ignoreTerms = cl.hasOption(ARG_IGNORE_TERMS);
verbose = cl.hasOption(ARG_VERBOSE);
setLogAppenders();
prepareDirectory();
streamManager = initializeStreamManager();
......@@ -140,7 +147,8 @@ public class Main {
Repositories repositories;
try {
repositories = new ReadRepositories("Fetching required repositories").readRepositories(directory, remoteOnly, ignoreGit, includeMaster);
repositories = new ReadRepositories("Fetching required repositories").readRepositories(directory,
remoteOnly, ignoreGit, includeMaster);
} catch (Exception ex) {
reportAndExit(-1);
return;
......@@ -151,7 +159,7 @@ public class Main {
boolean checkExamples = !ignoreExamples;
testRepository(repository, checkExamples);
}
for(Repository repository : repositories.getNamedRepositories()) {
for (Repository repository : repositories.getNamedRepositories()) {
boolean checkExamples = !ignoreExamples && remoteOnly;
testRepository(repository, checkExamples);
}
......@@ -160,20 +168,20 @@ public class Main {
new CheckConfig().doJob(dataset, ignoreExamples);
if (generateSite) {
new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory, verbose, ignoreExamples, ignoreTerms);
new GeneratePortal("Generate static files for the portal", streamManager).doJob(dataset, directory, verbose,
ignoreExamples, ignoreTerms);
}
if (repositories.getDefaultRepository() != null && !ignoreGit) {
Repository repository = repositories.getDefaultRepository();
resetCheckout(repository);
}
for(Repository repository : repositories.getNamedRepositories()) {
for (Repository repository : repositories.getNamedRepositories()) {
resetCheckout(repository);
}
testSuites.clean();
reportAndExit((int) -Math.signum(testSuites.getErrors() + testSuites.getFailures()));
}
......@@ -186,22 +194,26 @@ public class Main {
org.apache.log4j.Logger loggerBase = org.apache.log4j.Logger.getLogger(Constants.LOGGER_BASE);
TestSuitesAppender appender = new TestSuitesAppender(testSuites);
loggerBase.addAppender(appender);
}
}
private static void prepareDirectory() throws IOException {
target = new File(directory, Constants.TARGET_DIR);
File site = new File(directory, Constants.SITE_DIR);
FileUtils.forceMkdir(target);
if(generateSite) {
File staticTargetDir = new File(directory, Constants.STATIC_TARGET_DIR);
File staticTargetZip = new File(directory, Constants.STATIC_TARGET_DIR + ".zip");
URL url = Main.class.getClassLoader().getResource("static.zip");
FileUtils.copyURLToFile(url, staticTargetZip);
UnzipFile.unzip(staticTargetZip, staticTargetDir);
FileUtils.deleteQuietly(staticTargetZip);
if (generateSite) {
try (Git git = Git.cloneRepository().setURI(Constants.SAREF_PORTAL_STATIC_GIT).setDirectory(site).call()) {
LOG.info(String.format("Cloning site repository %s", Constants.SAREF_PORTAL_STATIC_GIT));
} catch (Exception ex) {
try (Git git = Git.open(site)) {
git.pull().call();
LOG.info(String.format("Pulling site repository %s", Constants.SAREF_PORTAL_STATIC_GIT));
} catch (Exception e) {
throw new IOException(String.format("Failed to clone or pull site repository %s", Constants.SAREF_PORTAL_STATIC_GIT), e);
}
}
}
}
private static Dataset createFreshDataset() throws IOException {
File datasetDir = new File(directory, Constants.DATASET_DIR);
FileUtils.forceMkdir(datasetDir);
......@@ -210,8 +222,10 @@ public class Main {
dataset.getDefaultModel().removeAll();
List<String> toRemove = new ArrayList<>();
Iterator<String> it = dataset.listNames();
while(it.hasNext()) { toRemove.add(it.next()); }
for(String name : toRemove) {
while (it.hasNext()) {
toRemove.add(it.next());
}
for (String name : toRemove) {
dataset.removeNamedModel(name);
}
dataset.addNamedModel(Constants.CONFIG, ModelFactory.createDefaultModel());
......@@ -225,7 +239,7 @@ public class Main {
String versionTestSuiteName = version.toString() + " - testing repository structure";
new CheckoutJob(versionTestSuiteName).checkoutVersion(version);
new CheckRepositoryStructure(versionTestSuiteName).check(version);
} catch(SAREFPipelineException ex) {
} catch (SAREFPipelineException ex) {
LOG.error("Error while testing repository structure " + version, ex);
continue;
}
......@@ -234,13 +248,13 @@ public class Main {
String ontologyTestSuiteName = version.toString() + " - testing ontology file";
testSuiteNames.put(version.getUri(), ontologyTestSuiteName);
new ReadOntology(ontologyTestSuiteName).doJob(dataset, version);
} catch(SAREFPipelineException ex) {
} catch (SAREFPipelineException ex) {
LOG.error(version.toString() + " Found errors for " + version.toString()
+ ". This version and all the examples will be ignored.", ex);
continue;
}
if(!checkExamples) {
if (!checkExamples) {
continue;
}
File examplesDir = new File(version.getRepository().getDirectory(), "examples");
......@@ -262,11 +276,11 @@ public class Main {
+ ". This example will be ignored.");
}
}
} catch(Exception ex) {
} catch (Exception ex) {
LOG.error(version.toString() + " Error while walking through the examples. They will be ignored.", ex);
continue;
continue;
}
}
}
}
private static void checkOWLProfile() {
......@@ -294,42 +308,53 @@ public class Main {
}
private static SPARQLExtStreamManager initializeStreamManager() throws IOException, URISyntaxException {
SPARQLExtStreamManager sm = SPARQLExtStreamManager.makeStreamManager();
LocationMapperAccept mapper = new LocationMapperAccept();
// set mapper for saref-pipeline src/documentation directory
URI uri = Main.class.getClassLoader().getResource("documentation").toURI();
boolean isJar = uri.getScheme().equals("jar");
Path dirPath;
if(uri.getScheme().equals("jar")) {
FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.<String, Object>emptyMap());
dirPath = fileSystem.getPath("/documentation");
if (isJar) {
FileSystem fileSystem = FileSystems.newFileSystem(uri, Collections.<String, Object>emptyMap());
dirPath = fileSystem.getPath("/documentation");
} else {
dirPath = Paths.get(uri);
}
LocatorFileAccept locator = new LocatorFileAccept(uri.getPath());
LocationMapperAccept mapper = new LocationMapperAccept();
SPARQLExtStreamManager sm = SPARQLExtStreamManager.makeStreamManager(locator);
walk(dirPath, isJar, mapper);
// override with local files in documentation directory
dirPath = Paths.get(new File(directory, "documentation").toURI());
walk(dirPath, false, mapper);
sm.setLocationMapper(mapper);
return sm;
}
private static void walk(Path dirPath, boolean isJar, LocationMapperAccept mapper) throws IOException {
Files.walk(dirPath).forEach((p) -> {
String relativePath = dirPath.relativize(p).toString().replace("\\", "/");
String fileurl = BASE_DOC + relativePath;
if(uri.getScheme().equals("jar")) {
if(isJar) {
mapper.addAltEntry(fileurl, p.toString().substring(1));
} else {
mapper.addAltEntry(fileurl, p.toString());
}
});
return sm;
}
private static void resetCheckout(Repository repository) {
try(Git git = Git.open(repository.getDirectory())) {
try (Git git = Git.open(repository.getDirectory())) {
String currentBranch = repository.getCurrentBranch();
if(currentBranch != null) {
if (currentBranch != null) {
git.checkout().setName(currentBranch).call();
}
} catch (IOException | GitAPIException ex) {
LOG.warn("Error while reseting repository " + repository, ex);
}
}
private static void reportAndExit(int code) {
try {
File report = new File(target, "report_output.xml");
......@@ -341,23 +366,24 @@ public class Main {
final StringWriter sw = new StringWriter();
jaxbMarshaller.marshal(testSuites, sw);
if(generateSite) {
if (generateSite) {
// generate the report.html
File reportHTML = new File(directory, Constants.SITE_DIR + File.separator + "report.html");
try (IndentedWriter writer = new IndentedWriter(new FileOutputStream(reportHTML));) {
Context context = ContextUtils.build(writer).setBase(Constants.BASE)
.setDebugTemplate(verbose).setStreamManager(streamManager).build();
Context context = ContextUtils.build(writer).setBase(Constants.BASE).setDebugTemplate(verbose)
.setStreamManager(streamManager).build();
BindingHashMap binding = new BindingHashMap();
binding.add(VAR_TEST_SUITES, NodeFactory.createLiteral(sw.toString()));
List<Binding> bindings = new ArrayList<>();
bindings.add(binding);
String query = IOUtils.toString(
streamManager.open(new LookUpRequest("documentation/report/main.rqg", SPARQLExt.MEDIA_TYPE)),
streamManager
.open(new LookUpRequest("documentation/report/main.rqg", SPARQLExt.MEDIA_TYPE)),
StandardCharsets.UTF_8);
RootPlan reportPlan = PlanFactory.create(query, BASE_DOC);
reportPlan.execTemplateStream(bindings, context);
}
if (openBrowser) {
if (Desktop.isDesktopSupported() && Desktop.getDesktop().isSupported(Desktop.Action.BROWSE)) {
Desktop.getDesktop().browse(reportHTML.toURI());
......
package fr.emse.gitlab.saref;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.io.FileUtils;
public class UnzipFile {
public static void unzip(File fileZip, File destDir) throws IOException {
byte[] buffer = new byte[1024];
ZipInputStream zis = new ZipInputStream(new FileInputStream(fileZip));
ZipEntry zipEntry = zis.getNextEntry();
while (zipEntry != null) {
File newFile = newFile(destDir, zipEntry);
if(zipEntry.isDirectory()) {
FileUtils.forceMkdir(newFile);
} else {
FileOutputStream fos = new FileOutputStream(newFile);
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
fos.close();
}
zipEntry = zis.getNextEntry();
}
zis.closeEntry();
zis.close();
}
public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException {
File destFile = new File(destinationDir, zipEntry.getName());
String destDirPath = destinationDir.getCanonicalPath();
String destFilePath = destFile.getCanonicalPath();
if (!destFilePath.startsWith(destDirPath + File.separator)) {
throw new IOException("Entry is outside of the target dir: " + zipEntry.getName());
}
return destFile;
}
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment