Commit 3f9c002f authored by carignani's avatar carignani
Browse files

parse examples and configurable tosca version

parent b2193b72

requirements.txt

+2 −1
Original line number Diff line number Diff line 
Flask==0.10.1

python-docx

tosca-parser
 
 No newline at end of file

src/config.py

+1 −1
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ def env_or_false(key): 
    except KeyError:

        return False



VERSION = "0.0.4"

VERSION = "0.0.5"



SECRET = env_or_false("TOSCAIE_SECRET") or 'super_secret_key'

src/doc2tosca.py

+213 −56
Original line number Diff line number Diff line
@@ -6,17 +6,19 @@ Generate tosca definitions from Docx specfication 
import sys

import os

import re

import traceback

from io import StringIO



import docx

from docx.table import Table

from docx.text.paragraph import Paragraph



BASE_FILENAME = "generated_etsi_nfv_sol001_{}_types.yaml"

BASE_FILENAME = "generated_etsi_nfv_sol001_{}_{}_types.yaml"

TOSCA_VERSION = "tosca_simple_yaml_1_2"

DEFAULT_TOSCA_VERSION = "tosca_simple_yaml_1_2"

SPEC_VERSION = "v2.6.1"



allowed_versions = ["v2.6.1","v2.6.3", "v2.7.1"]

allowed_versions = ["v2.6.1", "v2.6.3", "v2.7.1", "v2.8.1", "v3.3.1"]



MODEL_NAMES = ['vnfd', 'nsd', 'pnfd', 'common']
@@ -33,9 +35,58 @@ imports: 
data_types:

'''



MODELS = {}

EXAMPLES = {}



class Section():

    '''

    Defines a section of the base document

    '''



    def __init__(self, from_id, to_id, title):

        self.from_id = from_id

        self.to_id = to_id

        self.is_annex = title.strip().startswith("Annex")

        

        if not self.is_annex:

            cleaned_title = title.strip().split("\t")

            self.title = cleaned_title[1]

            self.number = int(cleaned_title[0])

        else:

            cleaned_title = title.strip().split(" ")

            self.title = " ".join(cleaned_title[3:])

            self.letter = cleaned_title[1]



    def __repr__(self):

        if self.is_annex:

            return "({}, Annex {}, {}-{})".format(self.title,self.letter, self.from_id, self.to_id)

        else:

            return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)



class Example():



    def __init__(self, filename, text):

        self.filename = filename

        self.text = text



def match_definition_incipit(txt):

    '''

    Returns tru if txt matches the incipit of a definition,

    identified by the word 'tosca' 

    '''

    return bool(re.match(r'^tosca\.[a-zA-Z\.:0-9\s]*$',txt.split("\n")[0].strip()))



def is_tosca_example(paragraph):

    '''

    Returns true when a table contains TOSCA definitions, i.e.

    the table contains just one cell and text starts with an

    empty space ' '

    '''

    txt = paragraph.text

    return \

        txt.startswith("tosca_definitions_version: ")

        # bool(re.match(r'^[a-zA-Z\.]*.yaml$', txt))



def is_tosca_def(table):

    '''

    Returns true when a table contains TOSCA definitions, i.e.
@@ -48,13 +99,13 @@ def is_tosca_def(table): 
        len(table.columns) == 1 and \

        match_definition_incipit(txt)    



def tosca_model_info(name, imports):

def tosca_model_info(name, version, imports):

    '''

    Returns a dictionary for the information on the model

    Returns a dictionary to hold information on the model

    '''

    return {

        'name' : name,

        'fn' : BASE_FILENAME.format(name),

        'fn' : BASE_FILENAME.format(version.replace(".","-"), name),

        'fd' : None,

        'imports' : imports,

        'buf' :  StringIO()
@@ -96,6 +147,35 @@ def find_sect(sect_to_find, start_idx, doc_content): 
    print("FOUND " + sect_to_find + " at " + str(start_idx))

    return start_idx



def is_lvl1_section_hdn(txt):

    ''' Returns true if txt is level 1 heading'''

    clean_txt = txt.strip()

    return bool(re.match(r'^[0-9]+\t[a-zA-Z\s]*$', clean_txt)) or \

            bool(re.match(r'^Annex[\s]*[A-Z]+[\s\t]+[a-zA-Z\s\(\)]*', clean_txt))



def find_all_sections(doc_content):

    '''

    Scans the body of the document to find level 1 sections

    Returns a list of Section

    '''

    sections = []



    start_indx = 0

    end_indx = 1



    while end_indx < len(doc_content):

        my_elem = doc_content[end_indx]

        if isinstance(my_elem, Paragraph) and is_lvl1_section_hdn(my_elem.text):

            if start_indx != 0:

                sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))

            

            start_indx = end_indx

            

        end_indx = end_indx + 1

    

    sections.append(Section(start_indx, end_indx-1, doc_content[start_indx].text))

    return sections



def write_table_to_file(tab, buf):

    '''

    Writes content of table t in utf-8 encoding to file F
@@ -140,28 +220,92 @@ def generate_tables_between(a_id, b_id, content, buf): 
            if txt.strip().startswith("Name") or txt.strip().startswith("Shorthand") or \

                txt.strip().startswith("tosca_def"):

                continue

            print("----- Filtered out: " + txt.split("\n")[0])

            if not len(tmp_elem.rows) == 1:

                print("       Rows count != 1 ")

            if not len(tmp_elem.columns) == 1:

                print("       Columns count != 1 ")

            if not match_definition_incipit(txt):

                print("       Regex != 1 ")

            # print("----- Filtered out: " + txt.split("\n")[0])

            #if not len(tmp_elem.rows) == 1:

                #print("       Rows count != 1 ")

            #if not len(tmp_elem.columns) == 1:

            #    print("       Columns count != 1 ")

            #if not match_definition_incipit(txt):

            #    print("       Regex != 1 ")

    return definitions_count



def dump_header(model_name, buf, spec_version=SPEC_VERSION, imports=None):

def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, tosca_version=DEFAULT_TOSCA_VERSION):

    '''

    Writes the header to the file for a specific model

    '''

    buf.write(HDR.format(

        tosca_version=TOSCA_VERSION,

        tosca_version=tosca_version,

        model=model_name,

        spec_version=spec_version,

        imports=imports))



MODELS = {}

def is_start_of_example(line: str):

    if not isinstance(line, str):

        raise ValueError("NOT A STRING")



    return line.startswith("tosca_definitions_version: ")



def is_body_of_example(line: str):

    return line.startswith("imports:") or\

            line.startswith("node_types:") or\

            line.startswith("topology_template:") or\

            line.startswith("description:") or\

            line.startswith("  ") or\

            line.startswith(" ") or\

            line == ""



def get_example_file_name(line: str):

    matches = re.search(r'[a-zA-Z0-9_]*.yaml', line)

    if matches is not None:

        return matches.group(0)

    return ""



def parse_all_examples(txt):



def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri'):

    res = []

    new_example = ""

    filename = ""



    for line in txt:



        if isinstance(line, Paragraph):

            linetext = str(line.text)

        elif isinstance(line, str):

            linetext = line

        else:

            continue



        if is_start_of_example(linetext):

            filename = get_example_file_name(previous_line) 

            if filename != "":

                new_example = "# " + filename + "\n" + linetext

            #new_example = "" + linetext

        elif new_example != "" and is_body_of_example(linetext):

            new_example = new_example + "\n" + linetext

        elif len(new_example) > 0:

            res.append(Example(filename, new_example))

            new_example = ""



        previous_line = linetext



    return res



def generate_examples_between(a_id, b_id, content, EXAMPLES):



    try:

        examples = parse_all_examples(content[a_id:b_id])

    except:

        track = traceback.format_exc()

        print(track)

        return 0



    for example in examples:

        EXAMPLES[example.filename] = example



    return len(examples)





def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri', tosca_version=DEFAULT_TOSCA_VERSION):

    '''

    Takes a filename or file object and loads the definition into the MODELS dictionary

    '''
@@ -174,6 +318,7 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri' 
            import_stmt = 'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_version) + import_stmt

        MODELS[mn] = tosca_model_info(

            mn,

            spec_version, 

            '- ' + import_stmt

        )
@@ -184,49 +329,35 @@ def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri' 
        raise ValueError("Cannot open the submitted Docx file")



    for m in MODELS:

        dump_header(

        generate_header(

            MODELS[m]['name'], 

            MODELS[m]['buf'],

            spec_version, 

            MODELS[m]['imports'])



    p_id = 0



    cur_sect = "0"



    CONTENT = get_content(sol_001)

    tables=0



    while p_id < len(CONTENT):

        elem = CONTENT[p_id]

        if isinstance(elem, Paragraph) and elem.text == "Foreword":

            break

        p_id = p_id + 1



    if p_id >= len(CONTENT):

        print("FOREWORD NOT FOUND")



    sect_6_id = find_sect("6\tVNFD TOSCA model", p_id, CONTENT)



    sect_7_id = find_sect("7\tNSD TOSCA model", sect_6_id, CONTENT)



    sect_8_id = find_sect("8\tPNFD TOSCA model", sect_7_id, CONTENT)



    sect_9_id = find_sect("9\tCommon Definitions", sect_8_id, CONTENT)



    annex_a_id = find_sect("Annex A (informative):", sect_9_id, CONTENT)

            MODELS[m]['imports'],

            tosca_version

        )



    count = generate_tables_between(sect_6_id, sect_7_id, CONTENT, MODELS['vnfd']['buf'])

    print("Printed " + str(count) + " types to " + "VNFD\n\n\n")

    content = get_content(sol_001)

    sections = find_all_sections(content)



    count = generate_tables_between(sect_7_id, sect_8_id, CONTENT, MODELS['nsd']['buf'])

    print("Printed " + str(count) + " types to " + "NSD\n\n\n")

    sections_to_models = {

        6 : 'vnfd',

        7 : 'nsd',

        8 : 'pnfd',

        9 : 'common'

    }



    count = generate_tables_between(sect_8_id, sect_9_id, CONTENT, MODELS['pnfd']['buf'])

    print("Printed " + str(count) + " types to " + "PNFD\n\n\n")

    for sect in sections:



    count = generate_tables_between(sect_9_id, annex_a_id, CONTENT, MODELS['common']['buf'])

    print("Printed " + str(count) + " types to " + "Common\n\n\n")

        if not sect.is_annex:

            if sect.number in sections_to_models.keys():

                model = sections_to_models[sect.number]

                count = generate_tables_between(sect.from_id, sect.to_id, content, MODELS[model]['buf'])

                print("Printed " + str(count) + " types to " + model)

        else:

            if sect.letter == "A":

                count = generate_examples_between(sect.from_id, sect.to_id, content, EXAMPLES)

                print("Printed " + str(count) + " types to " + "Annex " + sect.letter)





def print_to_files(prefix=None):
@@ -244,6 +375,31 @@ def print_to_files(prefix=None): 
        MODELS[m]['fd'].write('\n')

        MODELS[m]['fd'].close()



    for k in EXAMPLES:

        if prefix is not None:

            fn = os.path.join(prefix, EXAMPLES[k].filename)

        else:

            fn = EXAMPLES[k].filename

        print("Writing example file: " + fn)

        with open(fn, 'w') as newf:

            newf.write(EXAMPLES[k].text)

            newf.write("\n")

            newf.close()



def parse_version_from_filename(filename):

    '''

    Parses the version from the filename

    '''

    base_filename = os.path.basename(filename)



    if base_filename.startswith("gs_NFV-SOL001v"):         

        return "v" + base_filename.strip("gs_NFV-SOL001v") \

                .replace("0",".").strip(".").strip("p.docx")

    if base_filename.startswith("gs_nfv-sol001v"):         

        return "v" + base_filename.strip("gs_nfv-sol001v") \

                .replace("0",".").strip(".").strip("p.docx")

    return ""



if __name__ == "__main__":



    try:
@@ -253,7 +409,8 @@ if __name__ == "__main__": 
        print('Usage: doc2tosca <docx-with-tosca-definitions>')

        sys.exit(1)



    generate_templates(SOL001_FN)

    ver = parse_version_from_filename(SOL001_FN)

    generate_templates(SOL001_FN, spec_version=ver)



    print_to_files()

src/templates/home.html

+3 −0
Original line number Diff line number Diff line
@@ -87,6 +87,9 @@ For any other inquiry, contact <a href="mailto:cti_support@etsi.org">ETSI CTI</a 
			Custom version (overrides the selector):

			<input type="text" name="custom-doc-version">

			<br />

			Custom TOSCA version (default is <code>{{default_tosca_version}}</code>):

			<input type="text" name="custom-tosca-version">

			<br />

			Root of import statements:

			<select class="form-control form-control-sm" form="doc2tosca-form" name="yaml-root-path">

				<option value="uri">Forge URL</option>

src/test_doc2tosca.py

+263 −5
Original line number Diff line number Diff line
@@ -2,12 +2,270 @@ 
import doc2tosca as d2t



def test_match_definition_incipit():

	assert d2t.match_definition_incipit("tosca.") == True

	assert d2t.match_definition_incipit("tosca.definitions") == True

	assert d2t.match_definition_incipit("<node_template>") == False

    assert d2t.match_definition_incipit("tosca.")

    assert d2t.match_definition_incipit("tosca.definitions")

    assert not d2t.match_definition_incipit("<node_template>")



def  test_is_tosca_def():

	assert False

def test_parse_versione_from_filename():

    '''

    Test parsing version out of filenames

    '''

    assert d2t.parse_version_from_filename("gs_NFV-SOL001v020601p.docx") == "v2.6.1"

    assert d2t.parse_version_from_filename("gs_nfv-sol001v020801p.docx") == "v2.8.1"



def test_is_lvl1_section_hdn():

    

    assert d2t.is_lvl1_section_hdn("6\tVNFD TOSCA model")

    assert d2t.is_lvl1_section_hdn("Annex A (informative)")

    assert d2t.is_lvl1_section_hdn("Annex C (normative):\tConformance\t284")



def test_section_init():



    ssss = d2t.Section(0, 10, "6\tVNFD TOSCA model")

    assert ssss.number == 6

    assert ssss.title == "VNFD TOSCA model"

    assert not ssss.is_annex



    aaaa = d2t.Section(11, 20, "Annex A (informative): My title")

    assert aaaa.is_annex

    assert aaaa.title == "My title"

    assert aaaa.letter == "A"



def test_is_example_start():

    assert d2t.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")





txt = '''

The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:

tosca_definitions_version: tosca_simple_yaml_1_2

 

description: type definition of tosca.MyExampleNS

 

imports:

  - etsi_nfv_sol001_nsd_types.yaml  # all of TOSCA types as defined in ETSI GS NFVSOL 001

 

node_types:

  tosca.MyExampleNS:

    derived_from: tosca.nodes.nfv.NS

    properties:

      descriptor_id:

        type: string

        constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]

        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177

      designer:

        type: string

        constraints: [ valid_values: [ MyCompany] ]

        default: MyCompany

      name:

        type: string

        constraints: [ valid_values: [ ExampleService ] ]

        default: ExampleService

      version:

        type: string

        constraints: [ valid_values: [ '1.0' ] ]

        default: '1.0'

      invariant_id:

        type: string

        constraints: [ valid_values: [ 1111-2222-aaaa-bbbb ] ]

        default: 1111-2222-aaaa-bbbb

      flavour_id: 

        type: string

        constraints: [ valid_values: [ small, big ] ]

        default: small

    requirements:

      - virtual_link:

          capability: tosca.capabilities.nfv.VirtualLinkable

    interfaces:

      Nslcm:

        type: tosca.interfaces.nfv.Nslcm



The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.

MyExampleNS_2.yaml:

tosca_definitions_version: tosca_simple_yaml_1_1



description: Relational database, simple



imports:

  - etsi_nfv_sol001_nsd_types.yaml  # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001

  - example_VNF3.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF3, this file might be included in the NSD file structure

  - example_VNF4.yaml # uri of the yaml file which contains the definition of tosca.nodes.nfv.example_VNF4, this file might be included in the NSD file structure



node_types:

  tosca.myExample.NS_2:

    derived_from: tosca.nodes.nfv.NS

    properties:

      descriptor_id:

        type: string

        constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]

        default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574

      designer:

        type: string

        constraints: [ valid_values: [ MyCompany] ]

        default: MyCompany

      name:

        type: string

        constraints: [ valid_values: [ myExample2Service ] ]

        default: myExample2Service

      version:

        type: string

        constraints: [ valid_values: [ '1.0.0.0' ] ]

        default: '1.0.0.0'

      invariant_id:

        type: string

        constraints: [ valid_values: [ aaaa-bbbb-cccc-dddd ] ]

        default: aaaa-bbbb-cccc-dddd

      flavour_id: 

        type: string

        constraints: [ valid_values: [ simple ] ]

        default: simple

    

topology_template:

  substitution_mappings:

    node_type: tosca.myExample.NS_2

    requirements:

      virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of 

                                              # VNF_2 is exposed as the Sap



   node_templates:

    NS_2:

      type: tosca.myExample.NS_2

      interfaces:

        Nslcm:

          instantiate:

            implementation: instantiate.workflow.yaml

          terminate:

            implementation: terminate.workflow.yaml



    VNF_3:

      type: tosca.nodes.nfv.example_VNF3

      properties:

        # no property assignments needed for required properties that have a default value assigned in the node type definition, e.g. descriptor_id

        flavour_id: simple

        vnf_profile:

          instantiation_level: level_1

          min_number_of_instances: 2

          max_number_of_instances: 6

      requirements:

        - virtual_link: NS_VL_2

    

    VNF_4:

      type: tosca.nodes.nfv.example_VNF4

      properties:

        flavour_id: simple

        vnf_profile:

          instantiation_level: level_1

          min_number_of_instances: 1

          max_number_of_instances: 3

      requirements:

        - virtual_link_1: NS_VL_2

        # - virtual_link_2: # map to virtual_link requirement of the NS node



    NS_VL_2:

      type: tosca.nodes.nfv.NsVirtualLink

      properties:

        connectivity_type:

          layer_protocols: [ipv4]

          flow_pattern: mesh

        vl_profile:

          max_bitrate_requirements:

             root: 1000

          min_bitrate_requirements:    

             root: 1000

  



Arial A.13   Virtual IP address connection point

Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.

Load balancing

'''



txt_short = '''

The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:

tosca_definitions_version: tosca_simple_yaml_1_2

 

description: type definition of tosca.MyExampleNS

 

imports:

  - etsi_nfv_sol001_nsd_types.yaml  # all of TOSCA types as defined in ETSI GS NFVSOL 001

 

node_types:

  tosca.MyExampleNS:

    derived_from: tosca.nodes.nfv.NS

    properties:

      descriptor_id:

        type: string

        constraints: [ valid_values: [ b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ] ]

        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177

      designer:

        type: string

        constraints: [ valid_values: [ MyCompany] ]

        default: MyCompany



The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.

MyExampleNS_2.yaml:

tosca_definitions_version: tosca_simple_yaml_1_1



description: Relational database, simple



imports:

  - etsi_nfv_sol001_nsd_types.yaml  # all of NSD related TOSCA types as defined in ETSI GS NFV-SOL 001



node_types:

  tosca.myExample.NS_2:

    derived_from: tosca.nodes.nfv.NS

    properties:

      descriptor_id:

        type: string

        constraints: [ valid_values: [ c1bb0ab8-deab-4fa7-95ed-4840d70a3574 ] ]

        default: c1bb0ab8-deab-4fa7-95ed-4840d70a3574

    

topology_template:

  substitution_mappings:

    node_type: tosca.myExample.NS_2

    requirements:

      virtual_link: [ VNF_4, virtual_link_2 ] # the External connection point of 

                                              # VNF_2 is exposed as the Sap

    

    VNF_4:

      type: tosca.nodes.nfv.example_VNF4

      properties:

        flavour_id: simple

        vnf_profile:

          instantiation_level: level_1

          min_number_of_instances: 1

          max_number_of_instances: 3

      requirements:

        - virtual_link_1: NS_VL_2

        # - virtual_link_2: # map to virtual_link requirement of the NS node

  



Arial A.13   Virtual IP address connection point

Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.

Load balancing

'''

def test_parse_example_short():



    examples = d2t.parse_all_examples(txt_short.split("\n"))

    assert len(examples) == 2

    assert len(examples[1].text.split("\n")) == 37 # MyExampleNS_2.yaml





def test_parse_example_long():



    examples = d2t.parse_all_examples(txt.split("\n"))

    assert len(examples) == 2

    assert len(examples[1].text.split("\n")) == 93 # MyExampleNS_2.yaml







def test_get_example_file_name():



    line1 = """The following snippet shows the service template representing the NSD NS_2. In this example, NS_2 supports one single deployment flavour.

MyExampleNS_2.yaml:"""

    line2 = """The contents of MyExampleNs_Type.yaml file with the node type definition are as follows:"""

    line3 = """Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes."""





    assert d2t.get_example_file_name(line1) == "MyExampleNS_2.yaml"

    assert d2t.get_example_file_name(line2) == "MyExampleNs_Type.yaml"

    assert d2t.get_example_file_name(line3) == ""