Clause_9_5_Checker.java

/*
 * Copyright 2020 ETSI
 * 
 * Redistribution and use in source and binary forms, with or without 
 * modification, are permitted provided that the following conditions are met:
 * 1. Redistributions of source code must retain the above copyright notice, 
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice, 
 *    this list of conditions and the following disclaimer in the documentation 
 *    and/or other materials provided with the distribution.
 * 3. Neither the name of the copyright holder nor the names of its contributors 
 *    may be used to endorse or promote products derived from this software without 
 *    specific prior written permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
 * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package fr.emse.gitlab.saref.checkers;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.ArrayList;
import java.util.Optional;
import java.util.stream.Collectors;

import org.apache.any23.Any23;
import org.apache.any23.extractor.ExtractionException;
import org.apache.any23.source.DocumentSource;
import org.apache.any23.source.FileDocumentSource;
import org.apache.any23.writer.RDFXMLWriter;
import org.apache.any23.writer.TripleHandler;
import org.apache.any23.writer.TripleHandlerException;
import org.apache.commons.io.FileUtils;
import org.apache.jena.rdf.model.Model;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

import fr.emse.gitlab.saref.SAREFPipelineException;
import fr.emse.gitlab.saref.entities.SAREFRepository;
import fr.emse.gitlab.saref.entities.SAREFVersion;
import fr.emse.gitlab.saref.managers.DatasetManager;
import fr.emse.gitlab.saref.managers.GenerateRDFaManager;
import fr.emse.gitlab.saref.managers.RepositoryManager;
import fr.emse.gitlab.saref.managers.ThemisManager;
import okhttp3.MediaType;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;

/**
 * Checks TS 103 673 Clause 9.5: Ontology tests
 * 
 */
public class Clause_9_5_Checker extends AbstractClauseChecker {

	private static final PathMatcher csvMatcher = FileSystems.getDefault().getPathMatcher("glob:**/*.csv");
	private static final String FIRST_LINE = "Id;Requirement;Category;Test";

	private static enum MESSAGE {
		directories, csv, missing, ioexception, line, themis, themisError;
	}

	/**
	 * @param errorLogger
	 * @param config
	 */
	public Clause_9_5_Checker(RepositoryManager repositoryManager) {
		super(repositoryManager, Clause_9_5_Checker.class);
	}

	@Override
	public void checkClause() throws SAREFPipelineException {
		File dir = new File(repository.getDirectory(), "tests");
		if (!dir.isDirectory()) {
			return;
		}
		try {
			checkExists(dir.toPath());
			checkFirstLine(dir.toPath());
			testsRDFaGenerator();
			checkThemis();
		} catch (IOException ex) {
			logError(getMessage(MESSAGE.ioexception));
		}
	}

	private void checkExists(Path path) throws IOException {
		String directories = Files.walk(path).filter(p -> {
			try {
				return p.toFile().isDirectory() && !Files.isSameFile(path, p);
			} catch (IOException ex) {
				return false;
			}
		}).map(p -> p.toString()).collect(Collectors.joining(", "));
		if (directories.length() > 0) {
			logWarning(getMessage(MESSAGE.directories, directories));
		}

		String nonCsv = Files.walk(path, 1).filter(p -> {
			try {
				return p.toFile().isFile() && !csvMatcher.matches(p) && !p.toFile().getName().startsWith(".");
			} catch (Exception ex) {
				return false;
			}
		}).map(p -> p.getFileName().toString()).collect(Collectors.joining(", "));
		if (nonCsv.length() > 0) {
			logError(getMessage(MESSAGE.csv, nonCsv));
		}

		boolean containsFile = Files.walk(path, 1).anyMatch(p -> {
			return csvMatcher.matches(p);
		});
		if (!containsFile) {
			logWarning(getMessage(MESSAGE.missing));
		}
	}

	private void checkFirstLine(Path path) throws IOException {
		Files.walk(path).filter(p -> {
			return csvMatcher.matches(p);
		}).forEach(p -> {
			Optional<String> firstLine;
			try {
				firstLine = Files.lines(p).findFirst();
				if (!firstLine.isPresent()) {
					logError(getMessage(MESSAGE.line, p.getFileName()));
				} else {
					if (!firstLine.get().equals(FIRST_LINE)) {
						logError(getMessage(MESSAGE.line, p.getFileName()));
					}
				}
			} catch (IOException e) {
				logError(getMessage(MESSAGE.ioexception));
			}
		});
	}

	public void checkThemis() throws SAREFPipelineException {
		File versionSite = new File(siteManager.siteDir, version.getVersionPath());
		File tHTML = new File(versionSite, "tests.html");
		DocumentSource source = new FileDocumentSource(tHTML);
		Any23 runner = new Any23();
		try(ByteArrayOutputStream out = new ByteArrayOutputStream(); TripleHandler handler = new RDFXMLWriter(out);) {
			runner.extract(source, handler);

			String xmlData = out.toString();

			ArrayList<String[]> res = makeCall(xmlData, repository, datasetManager, version);

			ArrayList<String> result = new ArrayList<String>();

			for (int i = 0; i < res.size(); i++) {
				String response = res.get(i)[0] + ", " + res.get(i)[1] + ", " + res.get(i)[2];
				result.add(response);
			}

			if (!res.isEmpty()) {
				String data = result.stream().map(e -> e.toString()).collect(Collectors.joining("\n- ", "\n\n- ", "\n\n"));
				logWarning(getMessage(MESSAGE.themis, data));
			}

		} catch (ExtractionException | IOException | TripleHandlerException e) {
			logWarning(getMessage(MESSAGE.themisError));
		}
	}

	public ArrayList<String[]> makeCall(String xmlData, SAREFRepository repository, DatasetManager datasetManager,
			SAREFVersion version) throws SAREFPipelineException {

		ArrayList<String[]> res = new ArrayList<String[]>();
		try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
			String onto = "";

			Model model = version.getModel();
			model.write(out, "RDF/XML");
			onto = out.toString();

			OkHttpClient httpClient = new OkHttpClient().newBuilder().build();

			MediaType mediaType = MediaType.parse("application/json");

			String jsonRequest = "{\"ontologiesCode\":[\""
					+ onto.replaceAll("\\\"", "\\\\\"").replaceAll("\"", "\\\"").replaceAll("\t", " ") + "\"],"
					+ "\"testfile\":[\"" + xmlData.replaceAll("\\\"", "\\\\\"").replaceAll("\"", "\\\"").replaceAll("\t", " ")
					+ "\"]," + "\"format\":\"junit\"}";

			RequestBody body = RequestBody.create(jsonRequest, mediaType);

			Request request = new Request.Builder().url("http://themis.linkeddata.es/rest/api/results")
					.method("POST", body).addHeader("accept", "application/json")
					.addHeader("Content-Type", "application/json").build();

			Response response = httpClient.newCall(request).execute();

			String result = response.body().string();

			Document doc = ThemisManager.convertStringToXMLDocument(result);

			NodeList nodeList = doc.getElementsByTagName("testcase");
			for (int temp = 0; temp < nodeList.getLength(); temp++) {
				org.w3c.dom.Node node = nodeList.item(temp);
				String[] part = new String[3];
				if (node.getNodeType() == org.w3c.dom.Node.ELEMENT_NODE) {
					Element element = (Element) node;
					part[0] = "" + element.getAttributeNode("id");
					part[1] = "" + element.getAttributeNode("name");
					part[2] = "" + ((Element) node).getElementsByTagName("error").item(0).getAttributes()
							.getNamedItem("message");

					res.add(part);
				}
			}
		} catch (IOException e) {
			logWarning(getMessage(MESSAGE.themisError));
		}
		return res;
	}

	private void testsRDFaGenerator() throws SAREFPipelineException, IOException {
		if(pipeline.ignoreSite) {
			return;
		}
		String categoryChanger = "";
		String repoName = project.getName();
		String href = project.getNamespace();
		File testCSV = new File(repository.getDirectory(), "/tests/tests.csv");
		File versionSite = new File(siteManager.siteDir, version.getVersionPath());
		FileUtils.forceMkdir(versionSite);
		File testHTML = new File(versionSite, "tests.html");
		GenerateRDFaManager.GenerateRDFa(categoryChanger, repoName, href, testCSV, testHTML, "tests");
	}

}