From bd520050849ee8bb2789be0117143d60938de854 Mon Sep 17 00:00:00 2001
From: Michele Carignani <michele.carignani@etsi.org>
Date: Thu, 18 Mar 2021 11:56:18 +0100
Subject: [PATCH] attach logs to the generated zip file
---
src/doc2tosca.py | 39 ++++++++++++------------
src/web_app.py | 78 +++++++++++++++++++++++++++++++-----------------
2 files changed, 71 insertions(+), 46 deletions(-)
diff --git a/src/doc2tosca.py b/src/doc2tosca.py
index 861d320..6d1bfc7 100644
--- a/src/doc2tosca.py
+++ b/src/doc2tosca.py
@@ -6,6 +6,7 @@ Generate tosca definitions from Docx specfication
import sys
import os
import re
+import logging
from io import StringIO
import docx
@@ -141,9 +142,9 @@ def get_content(doc):
ret.append(Table(element, body))
table_count = table_count + 1
# else:
- # print("Non paragraph or table " + str(type(element)))
- print("Paragraphs: " + str(parag_count))
- print("Tables: " + str(table_count))
+ # logging.info("Non paragraph or table " + str(type(element)))
+ logging.info("Paragraphs: " + str(parag_count))
+ logging.info("Tables: " + str(table_count))
return ret
@@ -160,7 +161,7 @@ def find_sect(sect_to_find, start_idx, doc_content):
break
start_idx = start_idx + 1
- print("FOUND " + sect_to_find + " at " + str(start_idx))
+ logging.info("FOUND " + sect_to_find + " at " + str(start_idx))
return start_idx
@@ -249,7 +250,7 @@ def gen_tables_btwn(a_id, b_id, content, buf):
for idx in range(a_id, b_id):
if idx >= len(content):
- print(
+ logging.info(
range_err_mess.format(a_id, b_id, idx, len(content)))
return definitions_count
@@ -257,7 +258,7 @@ def gen_tables_btwn(a_id, b_id, content, buf):
if isinstance(tmp_elem, Paragraph) and \
is_lvl2_section_hdn(tmp_elem.text):
- print(tmp_elem.text)
+ logging.info(tmp_elem.text)
write_subsection_to_file(tmp_elem.text.split("\t")[1], buf)
definitions_count = definitions_count + 1
@@ -300,11 +301,11 @@ def init_models(yaml_root, spec_ver, tosc_ver):
spec_ver, import_stmt
)
- MODELS[model] = tosca_model_info(
- model,
- spec_ver,
- '- ' + import_stmt
- )
+ MODELS[model] = tosca_model_info(
+ model,
+ spec_ver,
+ '- ' + import_stmt
+ )
for mod in MODELS:
generate_header(
@@ -326,14 +327,14 @@ def generate_templates(
the MODELS dictionary
'''
if isinstance(filename, str):
- print("Opening " + filename)
+ logging.info("Opening " + filename)
init_models(yaml_root, spec_ver, tosc_ver)
try:
sol_001 = docx.Document(filename)
except:
- print("Error opening the submitted Docx file")
+ logging.info("Error opening the submitted Docx file")
raise ValueError("Cannot open the submitted Docx file")
content = get_content(sol_001)
@@ -347,13 +348,13 @@ def generate_templates(
count = gen_tables_btwn(
sect.from_id, sect.to_id, content, MODELS[model]['buf']
)
- print("Printed " + str(count) + " types to " + model)
+ logging.info("Printed " + str(count) + " types to " + model)
else:
if sect.letter == "A" or sect.letter == "E":
count = generate_examples_between(
sect.from_id, sect.to_id, content, EXAMPLES
)
- print("Printed {} types to Annex {}".format(
+ logging.info("Printed {} types to Annex {}".format(
str(count), sect.letter)
)
@@ -367,7 +368,7 @@ def print_to_files(prefix=None):
if prefix is not None:
mod['fn'] = os.path.join(prefix, mod['fn'])
- print("Writing to " + mod['fn'])
+ logging.info("Writing to " + mod['fn'])
mod['fd'] = open(mod['fn'], 'w')
mod['buf'].seek(0)
mod['fd'].write(mod['buf'].read())
@@ -379,7 +380,7 @@ def print_to_files(prefix=None):
fnm = os.path.join(prefix, "example_"+EXAMPLES[k].filename)
else:
fnm = EXAMPLES[k].filename
- print("Writing example file: " + fnm)
+ logging.info("Writing example file: " + fnm)
with open(fnm, 'w') as newf:
newf.write(EXAMPLES[k].text)
newf.write("\n")
@@ -410,8 +411,8 @@ if __name__ == "__main__":
try:
SOL001_FN = sys.argv[1]
except:
- print('Error: Filename missing or filename not a docx document')
- print('Usage: doc2tosca <docx-with-tosca-definitions>')
+ logging.info('Error: Filename missing or filename not a docx document')
+ logging.info('Usage: doc2tosca <docx-with-tosca-definitions>')
sys.exit(1)
ver = parse_version_from_filename(SOL001_FN)
diff --git a/src/web_app.py b/src/web_app.py
index d787300..913b75e 100644
--- a/src/web_app.py
+++ b/src/web_app.py
@@ -7,6 +7,8 @@ Web app to offer a frontend to the doc2tosca and tosca2doc tools
import os
import shutil
import tempfile
+import logging
+import traceback
from zipfile import ZipFile
from flask import Flask, flash, request, redirect, send_file, render_template, g
@@ -17,6 +19,7 @@ import config
import tosca2doc
import doc2tosca
+
class ReverseProxied(object):
def __init__(self, the_app, script_name=None, scheme=None, server=None):
@@ -40,6 +43,7 @@ class ReverseProxied(object):
environ['HTTP_HOST'] = server
return self.app(environ, start_response)
+
app = Flask(__name__)
app.wsgi_app = ReverseProxied(app.wsgi_app, script_name='/tosca-ie')
@@ -57,6 +61,7 @@ MISSING_MODULE_ERR_MSG = \
'Error: some TOSCA module missing. Make sure to \
upload definitions for NSD, VNFD, PNFD.'
+
def allowed_file(filename):
'''
Check filename is in the list of allowed extensions
@@ -64,6 +69,7 @@ def allowed_file(filename):
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+
@app.route("/")
def hello():
'''
@@ -75,14 +81,15 @@ def hello():
link = "#"
return render_template(
- "./home.html",
- VERSION=config.VERSION,
+ "./home.html",
+ VERSION=config.VERSION,
doc_allowed_versions=doc2tosca.allowed_versions,
default_tosca_version=doc2tosca.DEFAULT_TOSCA_VERSION,
last_rev=config.LAST_COMMIT,
rev_link=link
)
+
@app.route("/doc2tosca-info")
def doc2tosca_info():
'''
@@ -90,6 +97,7 @@ def doc2tosca_info():
'''
return render_template("./doc2tosca-details.html", VERSION=config.VERSION)
+
@app.route("/tosca2doc-info")
def tosca2doc_info():
'''
@@ -97,6 +105,7 @@ def tosca2doc_info():
'''
return render_template("./tosca2doc-details.html", VERSION=config.VERSION)
+
@app.route("/tosca2doc", methods=['POST'])
def mk_tosca2doc():
'''
@@ -107,14 +116,14 @@ def mk_tosca2doc():
flash('Error: No file uploaded.')
return redirect("/tosca-ie")
- ufiles=request.files.getlist("file")
+ ufiles = request.files.getlist("file")
for tosca_def in TOSCA_DEFS:
- found=False
+ found = False
for uploaded_file in ufiles:
if tosca_def in uploaded_file.filename or \
- tosca_def.lower() in uploaded_file.filename:
- found=True
+ tosca_def.lower() in uploaded_file.filename:
+ found = True
if not found:
flash(MISSING_MODULE_ERR_MSG)
print('Error: TOSCA module missing.')
@@ -129,7 +138,7 @@ def mk_tosca2doc():
tmp_dir = tempfile.mkdtemp()
g.tmp_dir = tmp_dir
- doc = docx.Document()
+ doc = docx.Document()
for tosca_def in TOSCA_DEFS:
for file in ufiles:
@@ -149,7 +158,8 @@ def mk_tosca2doc():
#return ("No content found")
-def get_all_yaml_file_paths(directory):
+
+def get_all_yaml_file_paths_and_logs(directory):
'''
Finds yaml files within a directory and sub directories and
returns the list of paths
@@ -159,12 +169,13 @@ def get_all_yaml_file_paths(directory):
for root, _, files in os.walk(directory):
for filename in files:
- if ".yaml" in filename:
+ if ".yaml" in filename or ".log" in filename:
filepath = os.path.join(root, filename)
file_paths.append(filepath)
return file_paths
+
@app.after_request
def after_request(response):
'''
@@ -175,16 +186,26 @@ def after_request(response):
if g.get('tmp_dir') is None:
return response
shutil.rmtree(g.tmp_dir, ignore_errors=True)
- print("Deleted {}\n\n".format(g.tmp_dir))
+ logging.debug("Deleted {}\n\n".format(g.tmp_dir))
return response
+
@app.route("/doc2tosca", methods=['POST'])
def mk_doc2tosca():
'''
Executes doc2tosca on the uploaded file
'''
+ tmp_dir = tempfile.mkdtemp()
+
+ logfilename = os.path.join(tmp_dir, 'toscatools.log')
+ logging.basicConfig(filename=logfilename, level=logging.INFO)
+
+ logging.info("Starting")
+ logging.debug("TMP DIR: " + tmp_dir)
+
zip_fn = "tosca_defs.zip"
+ zip_path = os.path.join(tmp_dir, zip_fn)
if 'file' not in request.files:
flash('No file uploaded')
@@ -193,7 +214,7 @@ def mk_doc2tosca():
ufiles = request.files.getlist("file")
try:
- print(request.form)
+ logging.debug(request.form)
doc_version = request.form.get('doc-version')
custom_doc_version = request.form.get('custom-doc-version')
custom_tosca_version = request.form.get('custom-tosca-version')
@@ -217,29 +238,32 @@ def mk_doc2tosca():
tosca_version = custom_tosca_version
try:
- doc2tosca.generate_templates(sol001_file, doc_version, yaml_root_path, tosca_version)
+ doc2tosca.generate_templates(
+ sol001_file, doc_version, yaml_root_path, tosca_version
+ )
+ doc2tosca.print_to_files(tmp_dir)
+
except ValueError as e:
flash(str(e))
return redirect("/tosca-ie")
except BaseException as e:
+ print(e)
+ # logging.error(traceback.print_exc())
flash(str(e))
- flash("Unknown error in the generation of the files. Please contact the support.")
- return redirect("/tosca-ie")
-
- tmp_dir = tempfile.mkdtemp()
- print("TMP DIR: " + tmp_dir)
- doc2tosca.print_to_files(tmp_dir)
+ flash("Unknown error in the generation of the files. \
+ Please contact the support.")
+ finally:
+ file_paths = get_all_yaml_file_paths_and_logs(tmp_dir)
- file_paths = get_all_yaml_file_paths(tmp_dir)
+ with ZipFile(zip_path, 'w') as archive:
+ for myfile in file_paths:
+ print(myfile)
+ logging.info(myfile)
+ archive.write(myfile, arcname=os.path.basename(myfile))
- zip_path = os.path.join(tmp_dir, zip_fn)
-
- with ZipFile( zip_path, 'w') as archive:
- for myfile in file_paths:
- archive.write(myfile, arcname=os.path.basename(myfile))
+ g.tmp_dir = tmp_dir
+ return send_file(zip_path, as_attachment=True)
- g.tmp_dir = tmp_dir
- return send_file(zip_path, as_attachment=True)
if __name__ == '__main__':
- app.run(debug=True,host='0.0.0.0')
+ app.run(debug=True, host='0.0.0.0')
--
GitLab