Skip to content
Snippets Groups Projects
Commit 3f9c002f authored by carignani's avatar carignani
Browse files

parse examples and configurable tosca version

parent b2193b72
No related branches found
No related tags found
No related merge requests found
Flask==0.10.1
python-docx
\ No newline at end of file
python-docx
tosca-parser
\ No newline at end of file
......@@ -13,7 +13,7 @@ def env_or_false(key):
except KeyError:
return False
VERSION = "0.0.4"
VERSION = "0.0.5"
SECRET = env_or_false("TOSCAIE_SECRET") or 'super_secret_key'
......@@ -6,17 +6,19 @@ Generate tosca definitions from Docx specfication
import sys
import os
import re
import traceback
from io import StringIO
import docx
from docx.table import Table
from docx.text.paragraph import Paragraph
BASE_FILENAME = "generated_etsi_nfv_sol001_{}_types.yaml"
BASE_FILENAME = "generated_etsi_nfv_sol001_{}_{}_types.yaml"
TOSCA_VERSION = "tosca_simple_yaml_1_2"
DEFAULT_TOSCA_VERSION = "tosca_simple_yaml_1_2"
SPEC_VERSION = "v2.6.1"
allowed_versions = ["v2.6.1","v2.6.3", "v2.7.1"]
allowed_versions = ["v2.6.1", "v2.6.3", "v2.7.1", "v2.8.1", "v3.3.1"]
MODEL_NAMES = ['vnfd', 'nsd', 'pnfd', 'common']
......@@ -33,10 +35,59 @@ imports:
data_types:
'''
MODELS = {}
EXAMPLES = {}
class Section():
'''
Defines a section of the base document
'''
def __init__(self, from_id, to_id, title):
self.from_id = from_id
self.to_id = to_id
self.is_annex = title.strip().startswith("Annex")
if not self.is_annex:
cleaned_title = title.strip().split("\t")
self.title = cleaned_title[1]
self.number = int(cleaned_title[0])
else:
cleaned_title = title.strip().split(" ")
self.title = " ".join(cleaned_title[3:])
self.letter = cleaned_title[1]
def __repr__(self):
if self.is_annex:
return "({}, Annex {}, {}-{})".format(self.title,self.letter, self.from_id, self.to_id)
else:
return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)
class Example():
def __init__(self, filename, text):
self.filename = filename
self.text = text
def match_definition_incipit(txt):
'''
Returns tru if txt matches the incipit of a definition,
identified by the word 'tosca'
'''
return bool(re.match(r'^tosca\.[a-zA-Z\.:0-9\s]*$',txt.split("\n")[0].strip()))
def is_tosca_def(table):
def is_tosca_example(paragraph):
'''
Returns true when a table contains TOSCA definitions, i.e.
the table contains just one cell and text starts with an
empty space ' '
'''
txt = paragraph.text
return \
txt.startswith("tosca_definitions_version: ")
# bool(re.match(r'^[a-zA-Z\.]*.yaml$', txt))
def is_tosca_def(table):
'''
Returns true when a table contains TOSCA definitions, i.e.
the table contains just one cell and text starts with an
......@@ -46,15 +97,15 @@ def is_tosca_def(table):
return \
len(table.rows) == 1 and \
len(table.columns) == 1 and \
match_definition_incipit(txt)
match_definition_incipit(txt)
def tosca_model_info(name, imports):
def tosca_model_info(name, version, imports):
'''
Returns a dictionary for the information on the model
Returns a dictionary to hold information on the model
'''
return {
'name' : name,
'fn' : BASE_FILENAME.format(name),
'fn' : BASE_FILENAME.format(version.replace(".","-"), name),
'fd' : None,
'imports' : imports,
'buf' : StringIO()
......@@ -96,6 +147,35 @@ def find_sect(sect_to_find, start_idx, doc_content):
print("FOUND " + sect_to_find + " at " + str(start_idx))
return start_idx
def is_lvl1_section_hdn(txt):
''' Returns true if txt is level 1 heading'''
clean_txt = txt.strip()
return bool(re.match(r'^[0-9]+\t[a-zA-Z\s]*$', clean_txt)) or \
bool(re.match(r'^Annex[\s]*[A-Z]+[\s\t]+[a-zA-Z\s\(\)]*', clean_txt))
def find_all_sections(doc_content):
'''
Scans the body of the document to find level 1 sections
Returns a list of Section
'''
sections = []
start_indx = 0
end_indx = 1
while end_indx < len(doc_content):
my_elem = doc_content[end_indx]
if isinstance(my_elem, Paragraph) and is_lvl1_section_hdn(my_elem.text):
if start_indx != 0:
sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))
start_indx = end_indx
end_indx = end_indx + 1
sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))
return sections
def write_table_to_file(tab, buf):
'''
Writes content of table t in utf-8 encoding to file F
......@@ -140,28 +220,92 @@ def generate_tables_between(a_id, b_id, content, buf):
if txt.strip().startswith("Name") or txt.strip().startswith("Shorthand") or \
txt.strip().startswith("tosca_def"):
continue
print("----- Filtered out: " + txt.split("\n")[0])
if not len(tmp_elem.rows) == 1:
print(" Rows count != 1 ")
if not len(tmp_elem.columns) == 1:
print(" Columns count != 1 ")
if not match_definition_incipit(txt):
print(" Regex != 1 ")
# print("----- Filtered out: " + txt.split("\n")[0])
#if not len(tmp_elem.rows) == 1:
#print(" Rows count != 1 ")
#if not len(tmp_elem.columns) == 1:
# print(" Columns count != 1 ")
#if not match_definition_incipit(txt):
# print(" Regex != 1 ")
return definitions_count
def dump_header(model_name, buf, spec_version=SPEC_VERSION, imports=None):
def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, tosca_version=DEFAULT_TOSCA_VERSION):
'''
Writes the header to the file for a specific model
'''
buf.write(HDR.format(
tosca_version=TOSCA_VERSION,
tosca_version=tosca_version,
model=model_name,
spec_version=spec_version,
imports=imports))
MODELS = {}
def is_start_of_example(line: str):
if not isinstance(line, str):
raise ValueError("NOT A STRING")
return line.startswith("tosca_definitions_version: ")
def is_body_of_example(line: str):
return line.startswith("imports:") or\
line.startswith("node_types:") or\
line.startswith("topology_template:") or\
line.startswith("description:") or\
line.startswith(" ") or\
line.startswith(" ") or\
line == ""
def get_example_file_name(line: str):
matches = re.search(r'[a-zA-Z0-9_]*.yaml', line)
if matches is not None:
return matches.group(0)
return ""
def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'):
def parse_all_examples(txt):
res = []
new_example = ""
filename = ""
for line in txt:
if isinstance(line, Paragraph):
linetext = str(line.text)
elif isinstance(line, str):
linetext = line
else:
continue
if is_start_of_example(linetext):
filename = get_example_file_name(previous_line)
if filename != "":
new_example = "# " + filename + "\n" + linetext
#new_example = "" + linetext
elif new_example != "" and is_body_of_example(linetext):
new_example = new_example + "\n" + linetext
elif len(new_example) > 0:
res.append(Example(filename, new_example))
new_example = ""
previous_line = linetext
return res
def generate_examples_between(a_id, b_id, content, EXAMPLES):
try:
examples = parse_all_examples(content[a_id:b_id])
except:
track = traceback.format_exc()
print(track)
return 0
for example in examples:
EXAMPLES[example.filename] = example
return len(examples)
def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri', tosca_version=DEFAULT_TOSCA_VERSION):
'''
Takes a filename or file object and loads the definition into the MODELS dictionary
'''
......@@ -173,7 +317,8 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
if yaml_root_path != 'local':
import_stmt = 'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_version) + import_stmt
MODELS[mn] = tosca_model_info(
mn,
mn,
spec_version,
'- ' + import_stmt
)
......@@ -184,49 +329,35 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
raise ValueError("Cannot open the submitted Docx file")
for m in MODELS:
dump_header(
generate_header(
MODELS[m]['name'],
MODELS[m]['buf'],
spec_version,
MODELS[m]['imports'])
p_id = 0
cur_sect = "0"
CONTENT = get_content(sol_001)
tables=0
while p_id < len(CONTENT):
elem = CONTENT[p_id]
if isinstance(elem, Paragraph) and elem.text == "Foreword":
break
p_id = p_id + 1
if p_id >= len(CONTENT):
print("FOREWORD NOT FOUND")
sect_6_id = find_sect("6\tVNFD TOSCA model", p_id, CONTENT)
sect_7_id = find_sect("7\tNSD TOSCA model", sect_6_id, CONTENT)
sect_8_id = find_sect("8\tPNFD TOSCA model", sect_7_id, CONTENT)
sect_9_id = find_sect("9\tCommon Definitions", sect_8_id, CONTENT)
annex_a_id = find_sect("Annex A (informative):", sect_9_id, CONTENT)
MODELS[m]['imports'],
tosca_version
)
count = generate_tables_between(sect_6_id, sect_7_id, CONTENT, MODELS['vnfd']['buf'])
print("Printed " + str(count) + " types to " + "VNFD\n\n\n")
content = get_content(sol_001)
sections = find_all_sections(content)
count = generate_tables_between(sect_7_id, sect_8_id, CONTENT, MODELS['nsd']['buf'])
print("Printed " + str(count) + " types to " + "NSD\n\n\n")
sections_to_models = {
6 : 'vnfd',
7 : 'nsd',
8 : 'pnfd',
9 : 'common'
}
count = generate_tables_between(sect_8_id, sect_9_id, CONTENT, MODELS['pnfd']['buf'])
print("Printed " + str(count) + " types to " + "PNFD\n\n\n")
for sect in sections:
count = generate_tables_between(sect_9_id, annex_a_id, CONTENT, MODELS['common']['buf'])
print("Printed " + str(count) + " types to " + "Common\n\n\n")
if not sect.is_annex:
if sect.number in sections_to_models.keys():
model = sections_to_models[sect.number]
count = generate_tables_between(sect.from_id, sect.to_id, content, MODELS[model]['buf'])
print("Printed " + str(count) + " types to " + model)
else:
if sect.letter == "A":
count = generate_examples_between(sect.from_id, sect.to_id, content, EXAMPLES)
print("Printed " + str(count) + " types to " + "Annex " + sect.letter)
def print_to_files(prefix=None):
......@@ -244,6 +375,31 @@ def print_to_files(prefix=None):
MODELS[m]['fd'].write('\n')
MODELS[m]['fd'].close()
for k in EXAMPLES:
if prefix is not None:
fn = os.path.join(prefix, EXAMPLES[k].filename)
else:
fn = EXAMPLES[k].filename
print("Writing example file: " + fn)
with open(fn, 'w') as newf:
newf.write(EXAMPLES[k].text)
newf.write("\n")
newf.close()
def parse_version_from_filename(filename):
'''
Parses the version from the filename
'''
base_filename = os.path.basename(filename)
if base_filename.startswith("gs_NFV-SOL001v"):
return "v" + base_filename.strip("gs_NFV-SOL001v") \
.replace("0",".").strip(".").strip("p.docx")
if base_filename.startswith("gs_nfv-sol001v"):
return "v" + base_filename.strip("gs_nfv-sol001v") \
.replace("0",".").strip(".").strip("p.docx")
return ""
if __name__ == "__main__":
try:
......@@ -253,7 +409,8 @@ if __name__ == "__main__":
print('Usage: doc2tosca <docx-with-tosca-definitions>')
sys.exit(1)
generate_templates(SOL001_FN)
ver = parse_version_from_filename(SOL001_FN)
generate_templates(SOL001_FN, spec_version=ver)
print_to_files()
......
......@@ -87,6 +87,9 @@ For any other inquiry, contact <a href="mailto:cti_support@etsi.org">ETSI CTI</a
Custom version (overrides the selector):
<input type="text" name="custom-doc-version">
<br />
Custom TOSCA version (default is <code>{{default_tosca_version}}</code>):
<input type="text" name="custom-tosca-version">
<br />
Root of import statements:
<select class="form-control form-control-sm" form="doc2tosca-form" name="yaml-root-path">
<option value="uri">Forge URL</option>
......
......@@ -2,12 +2,270 @@
import doc2tosca as d2t
def test_match_definition_incipit():
assert d2t.match_definition_incipit("tosca.") == True
assert d2t.match_definition_incipit("tosca.definitions") == True
assert d2t.match_definition_incipit("<node_template>") == False
assert d2t.match_definition_incipit("tosca.")
assert d2t.match_definition_incipit("tosca.definitions")
assert not d2t.match_definition_incipit("<node_template>")
def test_is_tosca_def():
assert False
def test_parse_versione_from_filename():
'''
Test parsing version out of filenames
'''
assert d2t.parse_version_from_filename("gs_NFV-SOL001v020601p.docx") == "v2.6.1"
assert d2t.parse_version_from_filename("gs_nfv-sol001v020801p.docx") == "v2.8.1"
def test_is_lvl1_section_hdn():
assert d2t.is_lvl1_section_hdn("6\tVNFD TOSCA model")
assert d2t.is_lvl1_section_hdn("Annex A (informative)")
assert d2t.is_lvl1_section_hdn("Annex C (normative):\tConformance\t284")
def test_section_init():
ssss = d2t.Section(0, 10, "6\tVNFD TOSCA model")
assert ssss.number == 6
assert ssss.title == "VNFD TOSCA model"
assert not ssss.is_annex
aaaa = d2t.Section(11, 20, "Annex A (informative): My title")
assert aaaa.is_annex
assert aaaa.title == "My title"
assert aaaa.letter == "A"
def test_is_example_start():
assert d2t.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")
txt = '''
The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:
tosca_definitions_version: tosca_simple_yaml_1_2
description: type definition of tosca.MyExampleNS
imports:
- etsi_nfv_sol001_nsd_types.yaml # all of TOSCA types as defined in ETSI GS NFVSOL 001
node_types:
tosca.MyExampleNS:
derived_from: tosca.nodes.nfv.NS
properties:
descriptor_id:
type: string
constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]
default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
designer:
type: string
constraints: [ valid_values: [ MyCompany] ]
default: MyCompany
name:
type: string
constraints: [ valid_values: [ ExampleService ] ]
default: ExampleService
version:
type: string
constraints: [ valid_values: [ '1.0' ] ]
default: '1.0'
invariant_id:
type: string
constraints: [ valid_values: [ 1111-2222-aaaa-bbbb ] ]
default: 1111-2222-aaaa-bbbb
flavour_id:
type: string
constraints: [ valid_values: [ small, big ] ]
default: small
requirements:
- virtual_link:
capability: tosca.capabilities.nfv.VirtualLinkable
interfaces:
Nslcm:
type: tosca.interfaces.nfv.Nslcm
The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.
MyExampleNS_2.yaml:
tosca_definitions_version: tosca_simple_yaml_1_1
description: Relational database, simple
imports:
- etsi_nfv_sol001_nsd_types.yaml # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001
- example_VNF3.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF3, this file might be included in the NSD file structure
- example_VNF4.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF4, this file might be included in the NSD file structure
node_types:
tosca.myExample.NS_2:
derived_from: tosca.nodes.nfv.NS
properties:
descriptor_id:
type: string
constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]
default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574
designer:
type: string
constraints: [ valid_values: [ MyCompany] ]
default: MyCompany
name:
type: string
constraints: [ valid_values: [ myExample2Service ] ]
default: myExample2Service
version:
type: string
constraints: [ valid_values: [ '1.0.0.0' ] ]
default: '1.0.0.0'
invariant_id:
type: string
constraints: [ valid_values: [ aaaa-bbbb-cccc-dddd ] ]
default: aaaa-bbbb-cccc-dddd
flavour_id:
type: string
constraints: [ valid_values: [ simple ] ]
default: simple
topology_template:
substitution_mappings:
node_type: tosca.myExample.NS_2
requirements:
virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of
# VNF_2 is exposed as the Sap
node_templates:
NS_2:
type: tosca.myExample.NS_2
interfaces:
Nslcm:
instantiate:
implementation: instantiate.workflow.yaml
terminate:
implementation: terminate.workflow.yaml
VNF_3:
type: tosca.nodes.nfv.example_VNF3
properties:
# no property assignments needed for required properties that have a default value assigned in the node type definition, e.g. descriptor_id
flavour_id: simple
vnf_profile:
instantiation_level: level_1
min_number_of_instances: 2
max_number_of_instances: 6
requirements:
- virtual_link: NS_VL_2
VNF_4:
type: tosca.nodes.nfv.example_VNF4
properties:
flavour_id: simple
vnf_profile:
instantiation_level: level_1
min_number_of_instances: 1
max_number_of_instances: 3
requirements:
- virtual_link_1: NS_VL_2
# - virtual_link_2: # map to virtual_link requirement of the NS node
NS_VL_2:
type: tosca.nodes.nfv.NsVirtualLink
properties:
connectivity_type:
layer_protocols: [ipv4]
flow_pattern: mesh
vl_profile:
max_bitrate_requirements:
root: 1000
min_bitrate_requirements:
root: 1000
Arial A.13 Virtual IP address connection point
Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.
Load balancing
'''
txt_short = '''
The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:
tosca_definitions_version: tosca_simple_yaml_1_2
description: type definition of tosca.MyExampleNS
imports:
- etsi_nfv_sol001_nsd_types.yaml # all of TOSCA types as defined in ETSI GS NFVSOL 001
node_types:
tosca.MyExampleNS:
derived_from: tosca.nodes.nfv.NS
properties:
descriptor_id:
type: string
constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]
default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
designer:
type: string
constraints: [ valid_values: [ MyCompany] ]
default: MyCompany
The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.
MyExampleNS_2.yaml:
tosca_definitions_version: tosca_simple_yaml_1_1
description: Relational database, simple
imports:
- etsi_nfv_sol001_nsd_types.yaml # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001
node_types:
tosca.myExample.NS_2:
derived_from: tosca.nodes.nfv.NS
properties:
descriptor_id:
type: string
constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]
default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574
topology_template:
substitution_mappings:
node_type: tosca.myExample.NS_2
requirements:
virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of
# VNF_2 is exposed as the Sap
VNF_4:
type: tosca.nodes.nfv.example_VNF4
properties:
flavour_id: simple
vnf_profile:
instantiation_level: level_1
min_number_of_instances: 1
max_number_of_instances: 3
requirements:
- virtual_link_1: NS_VL_2
# - virtual_link_2: # map to virtual_link requirement of the NS node
Arial A.13 Virtual IP address connection point
Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.
Load balancing
'''
def test_parse_example_short():
examples = d2t.parse_all_examples(txt_short.split("\n"))
assert len(examples) == 2
assert len(examples[1].text.split("\n")) == 37 # MyExampleNS_2.yaml
def test_parse_example_long():
examples = d2t.parse_all_examples(txt.split("\n"))
assert len(examples) == 2
assert len(examples[1].text.split("\n")) == 93 # MyExampleNS_2.yaml
def test_get_example_file_name():
line1 = """The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.
MyExampleNS_2.yaml:"""
line2 = """The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:"""
line3 = """Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes."""
assert d2t.get_example_file_name(line1) == "MyExampleNS_2.yaml"
assert d2t.get_example_file_name(line2) == "MyExampleNs_Type.yaml"
assert d2t.get_example_file_name(line3) == ""
......@@ -72,7 +72,8 @@ def hello():
return render_template(
"./home.html",
VERSION=config.VERSION,
doc_allowed_versions=doc2tosca.allowed_versions
doc_allowed_versions=doc2tosca.allowed_versions,
default_tosca_version=doc2tosca.DEFAULT_TOSCA_VERSION
)
@app.route("/doc2tosca-info")
......@@ -188,6 +189,7 @@ def mk_doc2tosca():
print(request.form)
doc_version = request.form.get('doc-version')
custom_doc_version = request.form.get('custom-doc-version')
custom_tosca_version = request.form.get('custom-tosca-version')
yaml_root_path = request.form.get('yaml-root-path')
except:
flash("Something went wrong :/")
......@@ -199,14 +201,21 @@ def mk_doc2tosca():
sol001_file = ufiles[0]
tosca_version = doc2tosca.DEFAULT_TOSCA_VERSION
if custom_doc_version != "":
doc_version = custom_doc_version
if custom_tosca_version != "":
tosca_version = custom_tosca_version
try:
doc2tosca.generate_templates(sol001_file, doc_version, yaml_root_path)
doc2tosca.generate_templates(sol001_file, doc_version, yaml_root_path, tosca_version)
except ValueError as e:
flash(str(e))
return redirect("/tosca-ie")
except:
except BaseException as e:
flash(str(e))
flash("Unknown error in the generation of the files. Please contact the support.")
return redirect("/tosca-ie")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment