parse examples and configurable tosca version

requirements.txt

 Flask==0.10.1
-python-docx
\ No newline at end of file
+python-docx
+tosca-parser
\ No newline at end of file

src/config.py

@@ -13,7 +13,7 @@ def env_or_false(key):
     except KeyError:
         return False
 
-VERSION = "0.0.4"
+VERSION = "0.0.5"
 
 SECRET = env_or_false("TOSCAIE_SECRET") or 'super_secret_key'
 

src/doc2tosca.py

@@ -6,17 +6,19 @@ Generate tosca definitions from Docx specfication
 import sys
 import os
 import re
+import traceback
 from io import StringIO
 
 import docx
 from docx.table import Table
 from docx.text.paragraph import Paragraph
 
-BASE_FILENAME = "generated_etsi_nfv_sol001_{}_types.yaml"
+BASE_FILENAME = "generated_etsi_nfv_sol001_{}_{}_types.yaml"
 TOSCA_VERSION = "tosca_simple_yaml_1_2"
+DEFAULT_TOSCA_VERSION = "tosca_simple_yaml_1_2"
 SPEC_VERSION = "v2.6.1"
 
-allowed_versions = ["v2.6.1","v2.6.3", "v2.7.1"]
+allowed_versions = ["v2.6.1", "v2.6.3", "v2.7.1", "v2.8.1", "v3.3.1"]
 
 MODEL_NAMES = ['vnfd', 'nsd', 'pnfd', 'common']
 
@@ -33,10 +35,59 @@ imports:
 data_types:
 '''
 
+MODELS = {}
+EXAMPLES = {}
+
+class Section():
+    '''
+    Defines a section of the base document
+    '''
+
+    def __init__(self, from_id, to_id, title):
+        self.from_id = from_id
+        self.to_id = to_id
+        self.is_annex = title.strip().startswith("Annex")
+        
+        if not self.is_annex:
+            cleaned_title = title.strip().split("\t")
+            self.title = cleaned_title[1]
+            self.number = int(cleaned_title[0])
+        else:
+            cleaned_title = title.strip().split(" ")
+            self.title = " ".join(cleaned_title[3:])
+            self.letter = cleaned_title[1]
+
+    def __repr__(self):
+        if self.is_annex:
+            return "({}, Annex {}, {}-{})".format(self.title,self.letter, self.from_id, self.to_id)
+        else:
+            return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)
+
+class Example():
+
+    def __init__(self, filename, text):
+        self.filename = filename
+        self.text = text
+
 def match_definition_incipit(txt):
+    '''
+    Returns tru if txt matches the incipit of a definition,
+    identified by the word 'tosca' 
+    '''
     return bool(re.match(r'^tosca\.[a-zA-Z\.:0-9\s]*$',txt.split("\n")[0].strip()))
 
-def  is_tosca_def(table):
+def is_tosca_example(paragraph):
+    '''
+    Returns true when a table contains TOSCA definitions, i.e.
+    the table contains just one cell and text starts with an
+    empty space ' '
+    '''
+    txt = paragraph.text
+    return \
+        txt.startswith("tosca_definitions_version: ")
+        # bool(re.match(r'^[a-zA-Z\.]*.yaml$', txt))
+
+def is_tosca_def(table):
     '''
     Returns true when a table contains TOSCA definitions, i.e.
     the table contains just one cell and text starts with an
@@ -46,15 +97,15 @@ def  is_tosca_def(table):
     return \
         len(table.rows) == 1 and \
         len(table.columns) == 1 and \
-        match_definition_incipit(txt)
+        match_definition_incipit(txt)    
 
-def tosca_model_info(name, imports):
+def tosca_model_info(name, version, imports):
     '''
-    Returns a dictionary for the information on the model
+    Returns a dictionary to hold information on the model
     '''
     return {
         'name' : name,
-        'fn' : BASE_FILENAME.format(name),
+        'fn' : BASE_FILENAME.format(version.replace(".","-"), name),
         'fd' : None,
         'imports' : imports,
         'buf' :  StringIO()
@@ -96,6 +147,35 @@ def find_sect(sect_to_find, start_idx, doc_content):
     print("FOUND " + sect_to_find + " at " + str(start_idx))
     return start_idx
 
+def is_lvl1_section_hdn(txt):
+    ''' Returns true if txt is level 1 heading'''
+    clean_txt = txt.strip()
+    return bool(re.match(r'^[0-9]+\t[a-zA-Z\s]*$', clean_txt)) or \
+            bool(re.match(r'^Annex[\s]*[A-Z]+[\s\t]+[a-zA-Z\s\(\)]*', clean_txt))
+
+def find_all_sections(doc_content):
+    '''
+    Scans the body of the document to find level 1 sections
+    Returns a list of Section
+    '''
+    sections = []
+
+    start_indx = 0
+    end_indx = 1
+
+    while end_indx < len(doc_content):
+        my_elem = doc_content[end_indx]
+        if isinstance(my_elem, Paragraph) and is_lvl1_section_hdn(my_elem.text):
+            if start_indx != 0:
+                sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))
+            
+            start_indx = end_indx
+            
+        end_indx = end_indx + 1
+    
+    sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))
+    return sections
+
 def write_table_to_file(tab, buf):
     '''
     Writes content of table t in utf-8 encoding to file F
@@ -140,28 +220,92 @@ def generate_tables_between(a_id, b_id, content, buf):
             if txt.strip().startswith("Name") or txt.strip().startswith("Shorthand") or \
                 txt.strip().startswith("tosca_def"):
                 continue
-            print("----- Filtered out: " + txt.split("\n")[0])
-            if not len(tmp_elem.rows) == 1:
-                print("       Rows count != 1 ")
-            if not len(tmp_elem.columns) == 1:
-                print("       Columns count != 1 ")
-            if not match_definition_incipit(txt):
-                print("       Regex != 1 ")
+            # print("----- Filtered out: " + txt.split("\n")[0])
+            #if not len(tmp_elem.rows) == 1:
+                #print("       Rows count != 1 ")
+            #if not len(tmp_elem.columns) == 1:
+            #    print("       Columns count != 1 ")
+            #if not match_definition_incipit(txt):
+            #    print("       Regex != 1 ")
     return definitions_count
 
-def dump_header(model_name, buf, spec_version=SPEC_VERSION, imports=None):
+def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, tosca_version=DEFAULT_TOSCA_VERSION):
     '''
     Writes the header to the file for a specific model
     '''
     buf.write(HDR.format(
-        tosca_version=TOSCA_VERSION,
+        tosca_version=tosca_version,
         model=model_name,
         spec_version=spec_version,
         imports=imports))
 
-MODELS = {}
+def is_start_of_example(line: str):
+    if not isinstance(line, str):
+        raise ValueError("NOT A STRING")
+
+    return line.startswith("tosca_definitions_version: ")
+
+def is_body_of_example(line: str):
+    return line.startswith("imports:") or\
+            line.startswith("node_types:") or\
+            line.startswith("topology_template:") or\
+            line.startswith("description:") or\
+            line.startswith("  ") or\
+            line.startswith(" ") or\
+            line == ""
+
+def get_example_file_name(line: str):
+    matches = re.search(r'[a-zA-Z0-9_]*.yaml', line)
+    if matches is not None:
+        return matches.group(0)
+    return ""
 
-def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'):
+def parse_all_examples(txt):
+
+    res = []
+    new_example = ""
+    filename = ""
+
+    for line in txt:
+
+        if isinstance(line, Paragraph):
+            linetext = str(line.text)
+        elif isinstance(line, str):
+            linetext = line
+        else:
+            continue
+
+        if is_start_of_example(linetext):
+            filename = get_example_file_name(previous_line) 
+            if filename != "":
+                new_example = "# " + filename + "\n" + linetext
+            #new_example = "" + linetext
+        elif new_example != "" and is_body_of_example(linetext):
+            new_example = new_example + "\n" + linetext
+        elif len(new_example) > 0:
+            res.append(Example(filename, new_example))
+            new_example = ""
+
+        previous_line = linetext
+
+    return res
+
+def generate_examples_between(a_id, b_id, content, EXAMPLES):
+
+    try:
+        examples = parse_all_examples(content[a_id:b_id])
+    except:
+        track = traceback.format_exc()
+        print(track)
+        return 0
+
+    for example in examples:
+        EXAMPLES[example.filename] = example
+
+    return len(examples)
+
+
+def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri', tosca_version=DEFAULT_TOSCA_VERSION):
     '''
     Takes a filename or file object and loads the definition into the MODELS dictionary
     '''
@@ -173,7 +317,8 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
         if yaml_root_path != 'local':
             import_stmt = 'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_version) + import_stmt
         MODELS[mn] = tosca_model_info(
-            mn, 
+            mn,
+            spec_version, 
             '- ' + import_stmt
         )
 
@@ -184,49 +329,35 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
         raise ValueError("Cannot open the submitted Docx file")
 
     for m in MODELS:
-        dump_header(
+        generate_header(
             MODELS[m]['name'], 
             MODELS[m]['buf'],
             spec_version, 
-            MODELS[m]['imports'])
-
-    p_id = 0
-
-    cur_sect = "0"
-
-    CONTENT = get_content(sol_001)
-    tables=0
-
-    while p_id < len(CONTENT):
-        elem = CONTENT[p_id]
-        if isinstance(elem, Paragraph) and elem.text == "Foreword":
-            break
-        p_id = p_id + 1
-
-    if p_id >= len(CONTENT):
-        print("FOREWORD NOT FOUND")
-
-    sect_6_id = find_sect("6\tVNFD TOSCA model", p_id, CONTENT)
-
-    sect_7_id = find_sect("7\tNSD TOSCA model", sect_6_id, CONTENT)
-
-    sect_8_id = find_sect("8\tPNFD TOSCA model", sect_7_id, CONTENT)
-
-    sect_9_id = find_sect("9\tCommon Definitions", sect_8_id, CONTENT)
-
-    annex_a_id = find_sect("Annex A (informative):", sect_9_id, CONTENT)
+            MODELS[m]['imports'],
+            tosca_version
+        )
 
-    count = generate_tables_between(sect_6_id, sect_7_id, CONTENT, MODELS['vnfd']['buf'])
-    print("Printed " + str(count) + " types to " + "VNFD\n\n\n")
+    content = get_content(sol_001)
+    sections = find_all_sections(content)
 
-    count = generate_tables_between(sect_7_id, sect_8_id, CONTENT, MODELS['nsd']['buf'])
-    print("Printed " + str(count) + " types to " + "NSD\n\n\n")
+    sections_to_models = {
+        6 : 'vnfd',
+        7 : 'nsd',
+        8 : 'pnfd',
+        9 : 'common'
+    }
 
-    count = generate_tables_between(sect_8_id, sect_9_id, CONTENT, MODELS['pnfd']['buf'])
-    print("Printed " + str(count) + " types to " + "PNFD\n\n\n")
+    for sect in sections:
 
-    count = generate_tables_between(sect_9_id, annex_a_id, CONTENT, MODELS['common']['buf'])
-    print("Printed " + str(count) + " types to " + "Common\n\n\n")
+        if not sect.is_annex:
+            if sect.number in sections_to_models.keys():
+                model = sections_to_models[sect.number]
+                count = generate_tables_between(sect.from_id, sect.to_id, content, MODELS[model]['buf'])
+                print("Printed " + str(count) + " types to " + model)
+        else:
+            if sect.letter == "A":
+                count = generate_examples_between(sect.from_id, sect.to_id, content, EXAMPLES)
+                print("Printed " + str(count) + " types to " + "Annex " + sect.letter)
 
 
 def print_to_files(prefix=None):
@@ -244,6 +375,31 @@ def print_to_files(prefix=None):
         MODELS[m]['fd'].write('\n')
         MODELS[m]['fd'].close()
 
+    for k in EXAMPLES:
+        if prefix is not None:
+            fn = os.path.join(prefix, EXAMPLES[k].filename)
+        else:
+            fn = EXAMPLES[k].filename
+        print("Writing example file: " + fn)
+        with open(fn, 'w') as newf:
+            newf.write(EXAMPLES[k].text)
+            newf.write("\n")
+            newf.close()
+
+def parse_version_from_filename(filename):
+    '''
+    Parses the version from the filename
+    '''
+    base_filename = os.path.basename(filename)
+
+    if base_filename.startswith("gs_NFV-SOL001v"):         
+        return "v" + base_filename.strip("gs_NFV-SOL001v") \
+                .replace("0",".").strip(".").strip("p.docx")
+    if base_filename.startswith("gs_nfv-sol001v"):         
+        return "v" + base_filename.strip("gs_nfv-sol001v") \
+                .replace("0",".").strip(".").strip("p.docx")
+    return ""
+
 if __name__ == "__main__":
 
     try:
@@ -253,7 +409,8 @@ if __name__ == "__main__":
         print('Usage: doc2tosca <docx-with-tosca-definitions>')
         sys.exit(1)
 
-    generate_templates(SOL001_FN)
+    ver = parse_version_from_filename(SOL001_FN)
+    generate_templates(SOL001_FN, spec_version=ver)
 
     print_to_files()
 

src/templates/home.html

@@ -87,6 +87,9 @@ For any other inquiry, contact <a href="mailto:cti_support@etsi.org">ETSI CTI</a
 			Custom version (overrides the selector):
 			<input type="text" name="custom-doc-version">
 			<br />
+			Custom TOSCA version (default is <code>{{default_tosca_version}}</code>):
+			<input type="text" name="custom-tosca-version">
+			<br />
 			Root of import statements:
 			<select class="form-control form-control-sm" form="doc2tosca-form" name="yaml-root-path">
 				<option value="uri">Forge URL</option>

src/test_doc2tosca.py

@@ -2,12 +2,270 @@
 import doc2tosca as d2t
 
 def test_match_definition_incipit():
-	assert d2t.match_definition_incipit("tosca.") == True
-	assert d2t.match_definition_incipit("tosca.definitions") == True
-	assert d2t.match_definition_incipit("<node_template>") == False
+    assert d2t.match_definition_incipit("tosca.")
+    assert d2t.match_definition_incipit("tosca.definitions")
+    assert not d2t.match_definition_incipit("<node_template>")
 
-def  test_is_tosca_def():
-	assert False
+def test_parse_versione_from_filename():
+    '''
+    Test parsing version out of filenames
+    '''
+    assert d2t.parse_version_from_filename("gs_NFV-SOL001v020601p.docx") == "v2.6.1"
+    assert d2t.parse_version_from_filename("gs_nfv-sol001v020801p.docx") == "v2.8.1"
 
+def test_is_lvl1_section_hdn():
+    
+    assert d2t.is_lvl1_section_hdn("6\tVNFD TOSCA model")
+    assert d2t.is_lvl1_section_hdn("Annex A (informative)")
+    assert d2t.is_lvl1_section_hdn("Annex C (normative):\tConformance\t284")
 
+def test_section_init():
+
+    ssss = d2t.Section(0, 10, "6\tVNFD TOSCA model")
+    assert ssss.number == 6
+    assert ssss.title == "VNFD TOSCA model"
+    assert not ssss.is_annex
+
+    aaaa = d2t.Section(11, 20, "Annex A (informative): My title")
+    assert aaaa.is_annex
+    assert aaaa.title == "My title"
+    assert aaaa.letter == "A"
+
+def test_is_example_start():
+    assert d2t.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")
+
+
+txt = '''
+The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:
+tosca_definitions_version: tosca_simple_yaml_1_2
+ 
+description: type definition of tosca.MyExampleNS
+ 
+imports:
+  - etsi_nfv_sol001_nsd_types.yaml  # all of TOSCA types as defined in ETSI GS NFVSOL 001
+ 
+node_types:
+  tosca.MyExampleNS:
+    derived_from: tosca.nodes.nfv.NS
+    properties:
+      descriptor_id:
+        type: string
+        constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]
+        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
+      designer:
+        type: string
+        constraints: [ valid_values: [ MyCompany] ]
+        default: MyCompany
+      name:
+        type: string
+        constraints: [ valid_values: [ ExampleService ] ]
+        default: ExampleService
+      version:
+        type: string
+        constraints: [ valid_values: [ '1.0' ] ]
+        default: '1.0'
+      invariant_id:
+        type: string
+        constraints: [ valid_values: [ 1111-2222-aaaa-bbbb ] ]
+        default: 1111-2222-aaaa-bbbb
+      flavour_id: 
+        type: string
+        constraints: [ valid_values: [ small, big ] ]
+        default: small
+    requirements:
+      - virtual_link:
+          capability: tosca.capabilities.nfv.VirtualLinkable
+    interfaces:
+      Nslcm:
+        type: tosca.interfaces.nfv.Nslcm
+
+The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.
+MyExampleNS_2.yaml:
+tosca_definitions_version: tosca_simple_yaml_1_1
+
+description: Relational database, simple
+
+imports:
+  - etsi_nfv_sol001_nsd_types.yaml  # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001
+  - example_VNF3.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF3, this file might be included in the NSD file structure
+  - example_VNF4.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF4, this file might be included in the NSD file structure
+
+node_types:
+  tosca.myExample.NS_2:
+    derived_from: tosca.nodes.nfv.NS
+    properties:
+      descriptor_id:
+        type: string
+        constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]
+        default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574
+      designer:
+        type: string
+        constraints: [ valid_values: [ MyCompany] ]
+        default: MyCompany
+      name:
+        type: string
+        constraints: [ valid_values: [ myExample2Service ] ]
+        default: myExample2Service
+      version:
+        type: string
+        constraints: [ valid_values: [ '1.0.0.0' ] ]
+        default: '1.0.0.0'
+      invariant_id:
+        type: string
+        constraints: [ valid_values: [ aaaa-bbbb-cccc-dddd ] ]
+        default: aaaa-bbbb-cccc-dddd
+      flavour_id: 
+        type: string
+        constraints: [ valid_values: [ simple ] ]
+        default: simple
+    
+topology_template:
+  substitution_mappings:
+    node_type: tosca.myExample.NS_2
+    requirements:
+      virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of 
+                                              # VNF_2 is exposed as the Sap
+
+   node_templates:
+    NS_2:
+      type: tosca.myExample.NS_2
+      interfaces:
+        Nslcm:
+          instantiate:
+            implementation: instantiate.workflow.yaml
+          terminate:
+            implementation: terminate.workflow.yaml
+
+    VNF_3:
+      type: tosca.nodes.nfv.example_VNF3
+      properties:
+        # no property assignments needed for required properties that have a default value assigned in the node type definition, e.g. descriptor_id
+        flavour_id: simple
+        vnf_profile:
+          instantiation_level: level_1
+          min_number_of_instances: 2
+          max_number_of_instances: 6
+      requirements:
+        - virtual_link: NS_VL_2
+    
+    VNF_4:
+      type: tosca.nodes.nfv.example_VNF4
+      properties:
+        flavour_id: simple
+        vnf_profile:
+          instantiation_level: level_1
+          min_number_of_instances: 1
+          max_number_of_instances: 3
+      requirements:
+        - virtual_link_1: NS_VL_2
+        # - virtual_link_2: # map to virtual_link requirement of the NS node
+
+    NS_VL_2:
+      type: tosca.nodes.nfv.NsVirtualLink
+      properties:
+        connectivity_type:
+          layer_protocols: [ipv4]
+          flow_pattern: mesh
+        vl_profile:
+          max_bitrate_requirements:
+             root: 1000
+          min_bitrate_requirements:    
+             root: 1000
+  
+
+Arial A.13   Virtual IP address connection point
+Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.
+Load balancing
+'''
+
+txt_short = '''
+The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:
+tosca_definitions_version: tosca_simple_yaml_1_2
+ 
+description: type definition of tosca.MyExampleNS
+ 
+imports:
+  - etsi_nfv_sol001_nsd_types.yaml  # all of TOSCA types as defined in ETSI GS NFVSOL 001
+ 
+node_types:
+  tosca.MyExampleNS:
+    derived_from: tosca.nodes.nfv.NS
+    properties:
+      descriptor_id:
+        type: string
+        constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]
+        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
+      designer:
+        type: string
+        constraints: [ valid_values: [ MyCompany] ]
+        default: MyCompany
+
+The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.
+MyExampleNS_2.yaml:
+tosca_definitions_version: tosca_simple_yaml_1_1
+
+description: Relational database, simple
+
+imports:
+  - etsi_nfv_sol001_nsd_types.yaml  # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001
+
+node_types:
+  tosca.myExample.NS_2:
+    derived_from: tosca.nodes.nfv.NS
+    properties:
+      descriptor_id:
+        type: string
+        constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]
+        default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574
+    
+topology_template:
+  substitution_mappings:
+    node_type: tosca.myExample.NS_2
+    requirements:
+      virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of 
+                                              # VNF_2 is exposed as the Sap
+    
+    VNF_4:
+