support clauses for examples filenames, refactoring and linting

src/doc2tosca.py

@@ -6,13 +6,14 @@ Generate tosca definitions from Docx specfication
 import sys
 import os
 import re
-import traceback
 from io import StringIO
 
 import docx
 from docx.table import Table
 from docx.text.paragraph import Paragraph
 
+from example import generate_examples_between
+
 BASE_FILENAME = "generated_etsi_nfv_sol001_{}_{}_types.yaml"
 TOSCA_VERSION = "tosca_simple_yaml_1_2"
 DEFAULT_TOSCA_VERSION = "tosca_simple_yaml_1_2"
@@ -60,14 +61,7 @@ class Section():
     def __repr__(self):
         if self.is_annex:
             return "({}, Annex {}, {}-{})".format(self.title,self.letter, self.from_id, self.to_id)
-        else:
-            return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)
-
-class Example():
-
-    def __init__(self, filename, text):
-        self.filename = filename
-        self.text = text
+        return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)
 
 def match_definition_incipit(txt):
     '''
@@ -76,17 +70,6 @@ def match_definition_incipit(txt):
     '''
     return bool(re.match(r'^tosca\.[a-zA-Z\.:0-9\s]*$',txt.split("\n")[0].strip()))
 
-def is_tosca_example(paragraph):
-    '''
-    Returns true when a table contains TOSCA definitions, i.e.
-    the table contains just one cell and text starts with an
-    empty space ' '
-    '''
-    txt = paragraph.text
-    return \
-        txt.startswith("tosca_definitions_version: ")
-        # bool(re.match(r'^[a-zA-Z\.]*.yaml$', txt))
-
 def is_tosca_def(table):
     '''
     Returns true when a table contains TOSCA definitions, i.e.
@@ -180,24 +163,13 @@ def write_table_to_file(tab, buf):
     '''
     Writes content of table t in utf-8 encoding to file F
     '''
-    def pad2 (txt):
-        if txt.startswith("   "):
-            return " " + txt
-        if txt.startswith("  "):
-            return "  " + txt
-        if txt.startswith(" "):
-            return " " + txt
-        return "  " + txt
-
     txt = tab.rows[0].cells[0].text
-    # print("#  Included in: " + tab.rows[0].cells[0].text.split("\n")[0])
-    buf.write("\n".join([x for x in txt.split("\n")]))
-    # buf.write('\n# -------------------- #\n')
+    buf.write(txt)
     if not txt.endswith('\n'):
         buf.write('\n')
     buf.write('\n')
 
-def generate_tables_between(a_id, b_id, content, buf):
+def gen_tables_btwn(a_id, b_id, content, buf):
     '''
     Loops over content and writes all tosca definitions to the
     fdesc file. Returns the number of written definitions
@@ -210,7 +182,7 @@ def generate_tables_between(a_id, b_id, content, buf):
             print("B: " + str(b_id))
             print("IDX: " + str(idx))
             print("LEN(CONTENT): " + str(len(content)))
-            return 
+            return definitions_count 
         tmp_elem = content[idx]
         if isinstance(tmp_elem, Table) and is_tosca_def(tmp_elem):
             write_table_to_file(tmp_elem, buf)
@@ -229,7 +201,12 @@ def generate_tables_between(a_id, b_id, content, buf):
             #    print("       Regex != 1 ")
     return definitions_count
 
-def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, tosca_version=DEFAULT_TOSCA_VERSION):
+def generate_header(
+        model_name, 
+        buf, 
+        spec_version=SPEC_VERSION, 
+        imports=None, 
+        tosca_version=DEFAULT_TOSCA_VERSION):
     '''
     Writes the header to the file for a specific model
     '''
@@ -239,86 +216,25 @@ def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, to
         spec_version=spec_version,
         imports=imports))
 
-def is_start_of_example(line: str):
-    if not isinstance(line, str):
-        raise ValueError("NOT A STRING")
-
-    return line.startswith("tosca_definitions_version: ")
-
-def is_body_of_example(line: str):
-    return line.startswith("imports:") or\
-            line.startswith("node_types:") or\
-            line.startswith("topology_template:") or\
-            line.startswith("description:") or\
-            line.startswith("  ") or\
-            line.startswith(" ") or\
-            line == ""
-
-def get_example_file_name(line: str):
-    matches = re.search(r'[a-zA-Z0-9_]*.yaml', line)
-    if matches is not None:
-        return matches.group(0)
-    return ""
-
-def parse_all_examples(txt):
-
-    res = []
-    new_example = ""
-    filename = ""
-
-    for line in txt:
-
-        if isinstance(line, Paragraph):
-            linetext = str(line.text)
-        elif isinstance(line, str):
-            linetext = line
-        else:
-            continue
-
-        if is_start_of_example(linetext):
-            filename = get_example_file_name(previous_line) 
-            if filename != "":
-                new_example = "# " + filename + "\n" + linetext
-            #new_example = "" + linetext
-        elif new_example != "" and is_body_of_example(linetext):
-            new_example = new_example + "\n" + linetext
-        elif len(new_example) > 0:
-            res.append(Example(filename, new_example))
-            new_example = ""
-
-        previous_line = linetext
-
-    return res
-
-def generate_examples_between(a_id, b_id, content, EXAMPLES):
-
-    try:
-        examples = parse_all_examples(content[a_id:b_id])
-    except:
-        track = traceback.format_exc()
-        print(track)
-        return 0
-
-    for example in examples:
-        EXAMPLES[example.filename] = example
-
-    return len(examples)
-
-
-def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri', tosca_version=DEFAULT_TOSCA_VERSION):
+def generate_templates(
+        filename, 
+        spec_ver=SPEC_VERSION, 
+        yaml_root='uri', 
+        tosc_ver=DEFAULT_TOSCA_VERSION):
     '''
     Takes a filename or file object and loads the definition into the MODELS dictionary
     '''
     if isinstance(filename, str):
         print("Opening " + filename)
 
-    for mn in MODEL_NAMES:
+    for mod in MODEL_NAMES:
         import_stmt = 'etsi_nfv_sol001_common_types.yaml'
-        if yaml_root_path != 'local':
-            import_stmt = 'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_version) + import_stmt
-        MODELS[mn] = tosca_model_info(
-            mn,
-            spec_version, 
+        if yaml_root != 'local':
+            import_stmt = \
+                'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_ver) + import_stmt
+        MODELS[mod] = tosca_model_info(
+            mod,
+            spec_ver, 
             '- ' + import_stmt
         )
 
@@ -328,13 +244,13 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
         print("Error opening the submitted Docx file")
         raise ValueError("Cannot open the submitted Docx file")
 
-    for m in MODELS:
+    for mod in MODELS:
         generate_header(
-            MODELS[m]['name'], 
-            MODELS[m]['buf'],
-            spec_version, 
-            MODELS[m]['imports'],
-            tosca_version
+            MODELS[mod]['name'], 
+            MODELS[mod]['buf'],
+            spec_ver, 
+            MODELS[mod]['imports'],
+            tosc_ver
         )
 
     content = get_content(sol_001)
@@ -352,7 +268,7 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'
         if not sect.is_annex:
             if sect.number in sections_to_models.keys():
                 model = sections_to_models[sect.number]
-                count = generate_tables_between(sect.from_id, sect.to_id, content, MODELS[model]['buf'])
+                count = gen_tables_btwn(sect.from_id, sect.to_id, content, MODELS[model]['buf'])
                 print("Printed " + str(count) + " types to " + model)
         else:
             if sect.letter == "A":
@@ -364,24 +280,25 @@ def print_to_files(prefix=None):
     '''
     Prefix is a path to a folder to work into
     '''
-    for m in MODELS:
-        if prefix != None:
-            MODELS[m]['fn'] = os.path.join(prefix, MODELS[m]['fn'])
+    for key in MODELS:
+        mod = MODELS[key]
+        if prefix is not None:
+            mod['fn'] = os.path.join(prefix, mod['fn'])
 
-        print("Writing to " + MODELS[m]['fn'])
-        MODELS[m]['fd'] = open(MODELS[m]['fn'], 'w')
-        MODELS[m]['buf'].seek(0)
-        MODELS[m]['fd'].write(MODELS[m]['buf'].read())
-        MODELS[m]['fd'].write('\n')
-        MODELS[m]['fd'].close()
+        print("Writing to " + mod['fn'])
+        mod['fd'] = open(mod['fn'], 'w')
+        mod['buf'].seek(0)
+        mod['fd'].write(mod['buf'].read())
+        mod['fd'].write('\n')
+        mod['fd'].close()
 
     for k in EXAMPLES:
         if prefix is not None:
-            fn = os.path.join(prefix, EXAMPLES[k].filename)
+            fnm = os.path.join(prefix, "example_"+EXAMPLES[k].filename)
         else:
-            fn = EXAMPLES[k].filename
-        print("Writing example file: " + fn)
-        with open(fn, 'w') as newf:
+            fnm = EXAMPLES[k].filename
+        print("Writing example file: " + fnm)
+        with open(fnm, 'w') as newf:
             newf.write(EXAMPLES[k].text)
             newf.write("\n")
             newf.close()
@@ -410,7 +327,7 @@ if __name__ == "__main__":
         sys.exit(1)
 
     ver = parse_version_from_filename(SOL001_FN)
-    generate_templates(SOL001_FN, spec_version=ver)
+    generate_templates(SOL001_FN, spec_ver=ver)
 
     print_to_files()
 

src/test_doc2tosca.py

 #!/bin/bash
 import doc2tosca as d2t
+import example as exp
 
 def test_match_definition_incipit():
     assert d2t.match_definition_incipit("tosca.")
@@ -32,7 +33,7 @@ def test_section_init():
     assert aaaa.letter == "A"
 
 def test_is_example_start():
-    assert d2t.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")
+    assert exp.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")
 
 
 txt = '''
@@ -242,19 +243,150 @@ Arial A.13   Virtual IP address connection point
 Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.
 Load balancing
 '''
+
+sunshine = '''
+sunshineVNF.yaml
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+description: Relational database, non-scalable
+
+imports:
+  - etsi_nfv_sol001_vnfd_types.yaml  # all of TOSCA VNFD types as defined in ETSI GS NFV-SOL 001
+
+data_types:
+  MyCompany.datatypes.nfv.VnfInstantiateAdditionalParameters:
+    derived_from: tosca.datatypes.nfv.VnfOperationAdditionalParameters
+    properties:
+      parameter_1:
+        type: string
+        required: true
+        default: value_1
+      parameter_2:
+        type: string
+        required: true
+        default: value_2
+
+node_types:
+  MyCompany.SunshineDB.1_0.1_0:
+    derived_from: tosca.nodes.nfv.VNF
+    properties:
+      descriptor_id:
+        type: string
+        constraints: [ equal: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ]
+        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
+      provider:
+        type: string
+        constraints: [ equal: MyCompany ]
+        default: MyCompany
+      product_name:
+        type: string
+        constraints: [ equal: SunshineDB ]
+        default: SunshineDB
+      software_version:
+        type: string
+        constraints: [ equal: '1.0' ]
+        default: '1.0'
+      descriptor_version:
+        type: string
+        constraints: [ equal: '1.0' ]
+        default: '1.0'
+      flavour_id: 
+        type: string
+        constraints: [ valid_values: [ simple, complex ] ]
+        default: simple
+      flavour_description: 
+        type: string
+        default: "" #empty string 
+      vnfm_info:
+        type: list
+        entry_schema:
+          type: string
+          constraints: [ equal: '0:MyCompany-1.0.0' ]
+        default: [ '0:MyCompany-1.0.0' ]
+    requirements:
+      - virtual_link:
+          capability: tosca.capabilities.nfv.VirtualLinkable
+          occurrences: [ 0, 0 ]
+      - virtual_link_backend:
+          capability: tosca.capabilities.nfv.VirtualLinkable
+          occurrences: [ 0, 1 ]
+      - virtual_link_service:
+          capability: tosca.capabilities.nfv.VirtualLinkable
+          occurrences: [ 0, 1 ]
+    interfaces:
+      Vnflcm:
+        type: tosca.interfaces.nfv.Vnflcm
+        operations:
+          instantiate:
+            inputs:
+              additional_parameters:
+                type: MyCompany.datatypes.nfv.VnfInstantiateAdditionalParameters
+    
+
+The vnf node template in the sunshine.vnfd.tosca.yaml file is abstract and is subject to substitution; the lower-level templates
+'''
+
+sunshine_vnfd = '''
+SunshineDB: VNFD-top level
+sunshine.vnfd.tosca.yaml 
+tosca_definitions_version: tosca_simple_yaml_1_3
+
+description: Relational database, non-scalable
+imports:
+  - etsi_nfv_sol001_vnfd_types.yaml  # all of TOSCA VNFD types as defined in ETSI GS NFV SOL 001
+  - sunshineVNF.yaml # contains the VNF node type definition
+
+
+topology_template:
+  inputs:
+    flavour_id:
+      type: string
+      description: VNF deployment flavour selected by the consumer. It is provided in the API       
+
+  node_templates: 
+    SunshineDB:
+        type: MyCompany.SunshineDB.1_0.1_0
+        properties:
+          flavour_id: { get_input: flavour_id }
+          descriptor_id: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177
+          provider: MyCompany
+          product_name: SunshineDB
+          software_version: '1.0'
+          descriptor_version: '1.0'
+          vnfm_info:
+            - '0:MyCompany-1.0.0'
+       # requirements:
+          #- virtual_link_backend # mapped in lower-level templates
+          #- virtual_link_service # mapped in lower-level templates
+        # get_input function would be used to process the template at run time to access the selected flavour id. If the TOSCA functions are not supported by VNFM, the above function may not be needed, what other mechanisms can be used by VNFM which provide the same function are out of scope of the present document.
+
+Something
+'''
+
 def test_parse_example_short():
 
-    examples = d2t.parse_all_examples(txt_short.split("\n"))
+    examples = exp.parse_all_examples(txt_short.split("\n"))
     assert len(examples) == 2
     assert len(examples[1].text.split("\n")) == 37 # MyExampleNS_2.yaml
 
 
 def test_parse_example_long():
 
-    examples = d2t.parse_all_examples(txt.split("\n"))
+    examples = exp.parse_all_examples(txt.split("\n"))
     assert len(examples) == 2
     assert len(examples[1].text.split("\n")) == 93 # MyExampleNS_2.yaml
 
+def test_parse_example_sunshine():
+
+    examples = exp.parse_all_examples(sunshine.split("\n"))
+    assert len(examples) == 1
+    assert len(examples[0].text.split("\n")) == 78
+
+def test_parse_example_sunshine_vnfd():
+
+    examples = exp.parse_all_examples(sunshine_vnfd.split("\n"))
+    assert len(examples) == 1
+    assert len(examples[0].text.split("\n")) == 32
 
 
 def test_get_example_file_name():