support clauses for examples filenames, refactoring and linting

import sys

import os

import re

import traceback

from io import StringIO

import docx

from docx.table import Table

from docx.text.paragraph import Paragraph

from example import generate_examples_between

BASE_FILENAME = "generated_etsi_nfv_sol001_{}_{}_types.yaml"

TOSCA_VERSION = "tosca_simple_yaml_1_2"

DEFAULT_TOSCA_VERSION = "tosca_simple_yaml_1_2"

    def __repr__(self):

        if self.is_annex:

            return "({}, Annex {}, {}-{})".format(self.title,self.letter, self.from_id, self.to_id)

        else:

        return "({}, {}, {}-{})".format(self.title, self.number, self.from_id, self.to_id)

class Example():

    def __init__(self, filename, text):

        self.filename = filename

        self.text = text

def match_definition_incipit(txt):

    '''

    Returns tru if txt matches the incipit of a definition,

    '''

    return bool(re.match(r'^tosca\.[a-zA-Z\.:0-9\s]*$',txt.split("\n")[0].strip()))

def is_tosca_example(paragraph):

    '''

    Returns true when a table contains TOSCA definitions, i.e.

    the table contains just one cell and text starts with an

    empty space ' '

    '''

    txt = paragraph.text

    return \

        txt.startswith("tosca_definitions_version: ")

        # bool(re.match(r'^[a-zA-Z\.]*.yaml$', txt))

def is_tosca_def(table):

    '''

    Returns true when a table contains TOSCA definitions, i.e.

    '''

    Writes content of table t in utf-8 encoding to file F

    '''

    def pad2 (txt):

        if txt.startswith("   "):

            return " " + txt

        if txt.startswith("  "):

            return "  " + txt

        if txt.startswith(" "):

            return " " + txt

        return "  " + txt

    txt = tab.rows[0].cells[0].text

    # print("#  Included in: " + tab.rows[0].cells[0].text.split("\n")[0])

    buf.write("\n".join([x for x in txt.split("\n")]))

    # buf.write('\n# -------------------- #\n')

    buf.write(txt)

    if not txt.endswith('\n'):

        buf.write('\n')

    buf.write('\n')

def generate_tables_between(a_id, b_id, content, buf):

def gen_tables_btwn(a_id, b_id, content, buf):

    '''

    Loops over content and writes all tosca definitions to the

    fdesc file. Returns the number of written definitions

            print("B: " + str(b_id))

            print("IDX: " + str(idx))

            print("LEN(CONTENT): " + str(len(content)))

            return 

            return definitions_count 

        tmp_elem = content[idx]

        if isinstance(tmp_elem, Table) and is_tosca_def(tmp_elem):

            write_table_to_file(tmp_elem, buf)

            #    print("       Regex != 1 ")

    return definitions_count

def generate_header(model_name, buf, spec_version=SPEC_VERSION, imports=None, tosca_version=DEFAULT_TOSCA_VERSION):

def generate_header(

        model_name, 

        buf, 

        spec_version=SPEC_VERSION, 

        imports=None, 

        tosca_version=DEFAULT_TOSCA_VERSION):

    '''

    Writes the header to the file for a specific model

    '''

        spec_version=spec_version,

        imports=imports))

def is_start_of_example(line: str):

    if not isinstance(line, str):

        raise ValueError("NOT A STRING")

    return line.startswith("tosca_definitions_version: ")

def is_body_of_example(line: str):

    return line.startswith("imports:") or\

            line.startswith("node_types:") or\

            line.startswith("topology_template:") or\

            line.startswith("description:") or\

            line.startswith("  ") or\

            line.startswith(" ") or\

            line == ""

def get_example_file_name(line: str):

    matches = re.search(r'[a-zA-Z0-9_]*.yaml', line)

    if matches is not None:

        return matches.group(0)

    return ""

def parse_all_examples(txt):

    res = []

    new_example = ""

    filename = ""

    for line in txt:

        if isinstance(line, Paragraph):

            linetext = str(line.text)

        elif isinstance(line, str):

            linetext = line

        else:

            continue

        if is_start_of_example(linetext):

            filename = get_example_file_name(previous_line) 

            if filename != "":

                new_example = "# " + filename + "\n" + linetext

            #new_example = "" + linetext

        elif new_example != "" and is_body_of_example(linetext):

            new_example = new_example + "\n" + linetext

        elif len(new_example) > 0:

            res.append(Example(filename, new_example))

            new_example = ""

        previous_line = linetext

    return res

def generate_examples_between(a_id, b_id, content, EXAMPLES):

    try:

        examples = parse_all_examples(content[a_id:b_id])

    except:

        track = traceback.format_exc()

        print(track)

        return 0

    for example in examples:

        EXAMPLES[example.filename] = example

    return len(examples)

def generate_templates(filename, spec_version=SPEC_VERSION, yaml_root_path='uri', tosca_version=DEFAULT_TOSCA_VERSION):

def generate_templates(

        filename, 

        spec_ver=SPEC_VERSION, 

        yaml_root='uri', 

        tosc_ver=DEFAULT_TOSCA_VERSION):

    '''

    Takes a filename or file object and loads the definition into the MODELS dictionary

    '''

    if isinstance(filename, str):

        print("Opening " + filename)

    for mn in MODEL_NAMES:

    for mod in MODEL_NAMES:

        import_stmt = 'etsi_nfv_sol001_common_types.yaml'

        if yaml_root_path != 'local':

            import_stmt = 'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_version) + import_stmt

        MODELS[mn] = tosca_model_info(

            mn,

            spec_version, 

        if yaml_root != 'local':

            import_stmt = \

                'https://forge.etsi.org/rep/nfv/SOL001/raw/{}/'.format(spec_ver) + import_stmt

        MODELS[mod] = tosca_model_info(

            mod,

            spec_ver, 

            '- ' + import_stmt

        )

        print("Error opening the submitted Docx file")

        raise ValueError("Cannot open the submitted Docx file")

    for m in MODELS:

    for mod in MODELS:

        generate_header(

            MODELS[m]['name'], 

            MODELS[m]['buf'],

            spec_version, 

            MODELS[m]['imports'],

            tosca_version

            MODELS[mod]['name'], 

            MODELS[mod]['buf'],

            spec_ver, 

            MODELS[mod]['imports'],

            tosc_ver

        )

    content = get_content(sol_001)

        if not sect.is_annex:

            if sect.number in sections_to_models.keys():

                model = sections_to_models[sect.number]

                count = generate_tables_between(sect.from_id, sect.to_id, content, MODELS[model]['buf'])

                count = gen_tables_btwn(sect.from_id, sect.to_id, content, MODELS[model]['buf'])

                print("Printed " + str(count) + " types to " + model)

        else:

            if sect.letter == "A":

    '''

    Prefix is a path to a folder to work into

    '''

    for m in MODELS:

        if prefix != None:

            MODELS[m]['fn'] = os.path.join(prefix, MODELS[m]['fn'])

    for key in MODELS:

        mod = MODELS[key]

        if prefix is not None:

            mod['fn'] = os.path.join(prefix, mod['fn'])

        print("Writing to " + MODELS[m]['fn'])

        MODELS[m]['fd'] = open(MODELS[m]['fn'], 'w')

        MODELS[m]['buf'].seek(0)

        MODELS[m]['fd'].write(MODELS[m]['buf'].read())

        MODELS[m]['fd'].write('\n')

        MODELS[m]['fd'].close()

        print("Writing to " + mod['fn'])

        mod['fd'] = open(mod['fn'], 'w')

        mod['buf'].seek(0)

        mod['fd'].write(mod['buf'].read())

        mod['fd'].write('\n')

        mod['fd'].close()

    for k in EXAMPLES:

        if prefix is not None:

            fn = os.path.join(prefix, EXAMPLES[k].filename)

            fnm = os.path.join(prefix, "example_"+EXAMPLES[k].filename)

        else:

            fn = EXAMPLES[k].filename

        print("Writing example file: " + fn)

        with open(fn, 'w') as newf:

            fnm = EXAMPLES[k].filename

        print("Writing example file: " + fnm)

        with open(fnm, 'w') as newf:

            newf.write(EXAMPLES[k].text)

            newf.write("\n")

            newf.close()

        sys.exit(1)

    ver = parse_version_from_filename(SOL001_FN)

    generate_templates(SOL001_FN, spec_version=ver)

    generate_templates(SOL001_FN, spec_ver=ver)

    print_to_files()

#!/bin/bash

import doc2tosca as d2t

import example as exp

def test_match_definition_incipit():

    assert d2t.match_definition_incipit("tosca.")

    assert aaaa.letter == "A"

def test_is_example_start():

    assert d2t.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")

    assert exp.is_start_of_example("tosca_definitions_version: tosca_simple_yaml_1_1")

txt = '''

Virtual IP address connection points (VipCps) are used to allocate one or multiple IP addresses that are shared by other CP instances, which may be instances of the same or of different VduCp or VnfExtCp nodes.

Load balancing

'''

sunshine = '''

sunshineVNF.yaml

tosca_definitions_version: tosca_simple_yaml_1_3

description: Relational database, non-scalable

imports:

  - etsi_nfv_sol001_vnfd_types.yaml  # all of TOSCA VNFD types as defined in ETSI GS NFV-SOL 001

data_types:

  MyCompany.datatypes.nfv.VnfInstantiateAdditionalParameters:

    derived_from: tosca.datatypes.nfv.VnfOperationAdditionalParameters

    properties:

      parameter_1:

        type: string

        required: true

        default: value_1

      parameter_2:

        type: string

        required: true

        default: value_2

node_types:

  MyCompany.SunshineDB.1_0.1_0:

    derived_from: tosca.nodes.nfv.VNF

    properties:

      descriptor_id:

        type: string

        constraints: [ equal: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177 ]

        default: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177

      provider:

        type: string

        constraints: [ equal: MyCompany ]

        default: MyCompany

      product_name:

        type: string

        constraints: [ equal: SunshineDB ]

        default: SunshineDB

      software_version:

        type: string

        constraints: [ equal: '1.0' ]

        default: '1.0'

      descriptor_version:

        type: string

        constraints: [ equal: '1.0' ]

        default: '1.0'

      flavour_id: 

        type: string

        constraints: [ valid_values: [ simple, complex ] ]

        default: simple

      flavour_description: 

        type: string

        default: "" #empty string 

      vnfm_info:

        type: list

        entry_schema:

          type: string

          constraints: [ equal: '0:MyCompany-1.0.0' ]

        default: [ '0:MyCompany-1.0.0' ]

    requirements:

      - virtual_link:

          capability: tosca.capabilities.nfv.VirtualLinkable

          occurrences: [ 0, 0 ]

      - virtual_link_backend:

          capability: tosca.capabilities.nfv.VirtualLinkable

          occurrences: [ 0, 1 ]

      - virtual_link_service:

          capability: tosca.capabilities.nfv.VirtualLinkable

          occurrences: [ 0, 1 ]

    interfaces:

      Vnflcm:

        type: tosca.interfaces.nfv.Vnflcm

        operations:

          instantiate:

            inputs:

              additional_parameters:

                type: MyCompany.datatypes.nfv.VnfInstantiateAdditionalParameters

The vnf node template in the sunshine.vnfd.tosca.yaml file is abstract and is subject to substitution; the lower-level templates

'''

sunshine_vnfd = '''

SunshineDB: VNFD-top level

sunshine.vnfd.tosca.yaml 

tosca_definitions_version: tosca_simple_yaml_1_3

description: Relational database, non-scalable

imports:

  - etsi_nfv_sol001_vnfd_types.yaml  # all of TOSCA VNFD types as defined in ETSI GS NFV SOL 001

  - sunshineVNF.yaml # contains the VNF node type definition

topology_template:

  inputs:

    flavour_id:

      type: string

      description: VNF deployment flavour selected by the consumer. It is provided in the API       

  node_templates: 

    SunshineDB:

        type: MyCompany.SunshineDB.1_0.1_0

        properties:

          flavour_id: { get_input: flavour_id }

          descriptor_id: b1bb0ce7-ebca-4fa7-95ed-4840d70a1177

          provider: MyCompany

          product_name: SunshineDB

          software_version: '1.0'

          descriptor_version: '1.0'

          vnfm_info:

            - '0:MyCompany-1.0.0'

       # requirements:

          #- virtual_link_backend # mapped in lower-level templates

          #- virtual_link_service # mapped in lower-level templates

        # get_input function would be used to process the template at run time to access the selected flavour id. If the TOSCA functions are not supported by VNFM, the above function may not be needed, what other mechanisms can be used by VNFM which provide the same function are out of scope of the present document.

Something

'''

def test_parse_example_short():

    examples = d2t.parse_all_examples(txt_short.split("\n"))

    examples = exp.parse_all_examples(txt_short.split("\n"))

    assert len(examples) == 2

    assert len(examples[1].text.split("\n")) == 37 # MyExampleNS_2.yaml

def test_parse_example_long():

    examples = d2t.parse_all_examples(txt.split("\n"))

    examples = exp.parse_all_examples(txt.split("\n"))

    assert len(examples) == 2

    assert len(examples[1].text.split("\n")) == 93 # MyExampleNS_2.yaml

def test_parse_example_sunshine():

    examples = exp.parse_all_examples(sunshine.split("\n"))

    assert len(examples) == 1

    assert len(examples[0].text.split("\n")) == 78

def test_parse_example_sunshine_vnfd():

    examples = exp.parse_all_examples(sunshine_vnfd.split("\n"))

    assert len(examples) == 1

    assert len(examples[0].text.split("\n")) == 32

def test_get_example_file_name():