Commit d574194b authored by Lluis Gifre Renom's avatar Lluis Gifre Renom
Browse files

Common - Tools - RestConf Server: 

- Improve support for Yang Model Discovery
- Improve YANG path normalization to detect namespaces
- Enable Pre-Get data dispatchers
- Renamed old data dispatchers to update
- Update related test tools
parent 0ee4d415

src/common/tools/rest_conf/server/restconf_server/Callbacks.py

+38 −13
Original line number Diff line number Diff line
@@ -38,7 +38,22 @@ class _Callback: 
        '''

        return self._path_pattern.fullmatch(path)



    def execute_data(

    def execute_data_pre_get(

        self, match : re.Match, path : str, old_data : Optional[Dict]

    ) -> bool:

        '''

        Execute the callback action for a matched data path.

        This method should be implemented for each specific callback.

        @param match: `re.Match` object returned by `match()`.

        @param path: Original request path that was matched.

        @param old_data: Resource representation before retrieval, if applicable, otherwise `None`

        @returns boolean indicating whether additional callbacks should be executed, defaults to False

        '''

        MSG = 'match={:s}, path={:s}, old_data={:s}'

        msg = MSG.format(match.groupdict(), path, old_data)

        raise NotImplementedError(msg)



    def execute_data_update(

        self, match : re.Match, path : str, old_data : Optional[Dict],

        new_data : Optional[Dict]

    ) -> bool:
@@ -78,14 +93,24 @@ class CallbackDispatcher: 
    def register(self, callback : _Callback) -> None:

        self._callbacks.append(callback)



    def dispatch_data(

    def dispatch_data_pre_get(

        self, path : str, old_data : Optional[Dict] = None

    ) -> None:

        LOGGER.warning('[dispatch_data_pre_get] Checking Callbacks for path={:s}'.format(str(path)))

        for callback in self._callbacks:

            match = callback.match(path)

            if match is None: continue

            keep_running_callbacks = callback.execute_data_pre_get(match, path, old_data)

            if not keep_running_callbacks: break



    def dispatch_data_update(

        self, path : str, old_data : Optional[Dict] = None, new_data : Optional[Dict] = None

    ) -> None:

        LOGGER.warning('[dispatch_data] Checking Callbacks for path={:s}'.format(str(path)))

        LOGGER.warning('[dispatch_data_update] Checking Callbacks for path={:s}'.format(str(path)))

        for callback in self._callbacks:

            match = callback.match(path)

            if match is None: continue

            keep_running_callbacks = callback.execute_data(match, path, old_data, new_data)

            keep_running_callbacks = callback.execute_data_update(match, path, old_data, new_data)

            if not keep_running_callbacks: break



    def dispatch_operation(
@@ -113,7 +138,7 @@ class CallbackOnNetwork(_Callback): 
        pattern += r'/ietf-network:networks/network=(?P<network_id>[^/]+)'

        super().__init__(pattern)



    def execute_data(

    def execute_data_update(

        self, match : re.Match, path : str, old_data : Optional[Dict],

        new_data : Optional[Dict]

    ) -> bool:
@@ -127,7 +152,7 @@ class CallbackOnNode(_Callback): 
        pattern += r'/node=(?P<node_id>[^/]+)'

        super().__init__(pattern)



    def execute_data(

    def execute_data_update(

        self, match : re.Match, path : str, old_data : Optional[Dict],

        new_data : Optional[Dict]

    ) -> bool:
@@ -141,7 +166,7 @@ class CallbackOnLink(_Callback): 
        pattern += r'/ietf-network-topology:link=(?P<link_id>[^/]+)'

        super().__init__(pattern)



    def execute_data(

    def execute_data_update(

        self, match : re.Match, path : str, old_data : Optional[Dict],

        new_data : Optional[Dict]

    ) -> bool:
@@ -167,12 +192,12 @@ def main() -> None: 
    callbacks.register(CallbackOnLink())

    callbacks.register(CallbackShutdown())



    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin')

    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin/node=P-PE2')

    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin/ietf-network-topology:link=L6')

    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin/')

    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin/node=P-PE1/')

    callbacks.dispatch_data('/restconf/data/ietf-network:networks/network=admin/ietf-network-topology:link=L4/')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin/node=P-PE2')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin/ietf-network-topology:link=L6')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin/')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin/node=P-PE1/')

    callbacks.dispatch_data_update('/restconf/data/ietf-network:networks/network=admin/ietf-network-topology:link=L4/')

    callbacks.dispatch_operation('/restconf/operations/shutdown/')



if __name__ == '__main__':

src/common/tools/rest_conf/server/restconf_server/DispatchData.py

+9 −4
Original line number Diff line number Diff line
@@ -31,6 +31,11 @@ class RestConfDispatchData(Resource): 
        self._callback_dispatcher = callback_dispatcher



    def get(self, subpath : str = '/') -> Response:

        data = self._yang_handler.get(subpath)

        self._callback_dispatcher.dispatch_data_pre_get(

            '/restconf/data/' + subpath, old_data=data

        )



        data = self._yang_handler.get(subpath)

        if data is None:

            abort(
@@ -70,7 +75,7 @@ class RestConfDispatchData(Resource): 


        LOGGER.info('[POST] {:s} {:s} => {:s}'.format(subpath, str(payload), str(json_data)))



        self._callback_dispatcher.dispatch_data(

        self._callback_dispatcher.dispatch_data_update(

            '/restconf/data/' + subpath, old_data=None, new_data=json_data

        )
@@ -102,7 +107,7 @@ class RestConfDispatchData(Resource): 
        diff_data = deepdiff.DeepDiff(old_data, new_data)

        updated = len(diff_data) > 0



        self._callback_dispatcher.dispatch_data(

        self._callback_dispatcher.dispatch_data_update(

            '/restconf/data/' + subpath, old_data=old_data, new_data=new_data

        )
@@ -140,7 +145,7 @@ class RestConfDispatchData(Resource): 
        #diff_data = deepdiff.DeepDiff(old_data, new_data)

        #updated = len(diff_data) > 0



        self._callback_dispatcher.dispatch_data(

        self._callback_dispatcher.dispatch_data_update(

            '/restconf/data/' + subpath, old_data=old_data, new_data=new_data

        )
@@ -170,7 +175,7 @@ class RestConfDispatchData(Resource): 
                description='Path({:s}) not found'.format(str(subpath))

            )



        self._callback_dispatcher.dispatch_data(

        self._callback_dispatcher.dispatch_data_update(

            '/restconf/data/' + subpath, old_data=old_data, new_data=None

        )

src/common/tools/rest_conf/server/restconf_server/RestConfServerApplication.py

+3 −0
Original line number Diff line number Diff line
@@ -63,6 +63,9 @@ class RestConfServerApplication: 
        self._app.after_request(log_request)

        self._api = Api(self._app)



    @property

    def yang_handler(self): return self._yang_handler



    @property

    def callback_dispatcher(self): return self._callback_dispatcher

src/common/tools/rest_conf/server/restconf_server/YangHandler.py

+16 −3
Original line number Diff line number Diff line
@@ -54,6 +54,12 @@ class YangHandler: 
            json.dumps(yang_startup_data), fmt='json'

        )



    @property

    def yang_context(self): return self._yang_context



    @property

    def yang_datastore(self): return self._datastore



    def destroy(self) -> None:

        self._yang_context.destroy()
@@ -165,13 +171,20 @@ class YangHandler: 
                name, val = part.split('=', 1)

                # keep original name (may include prefix) for output, but

                # use local name (without module prefix) to lookup schema

                local_name = name.split(':', 1)[1] if ':' in name else name

                local_name = name #.split(':', 1)[1] if ':' in name else name

                schema_path = schema_path + '/' + local_name if schema_path else '/' + local_name

                schema_nodes = list(self._yang_context.find_path(schema_path))

                if len(schema_nodes) != 1:

                    MSG = 'No/Multiple SchemaNodes({:s}) for SchemaPath({:s})'

                    raise Exception(MSG.format(

                        str([repr(sn) for sn in schema_nodes]), schema_path

                        #str([repr(sn) for sn in schema_nodes]), schema_path

                        str([

                            '{:s}({:s}) => {:s}'.format(

                                repr(sn),

                                str(sn.schema_path()),

                                str([repr(snn) for snn in sn.iter_tree()])

                            )

                            for sn in schema_nodes]), schema_path

                    ))

                schema_node = schema_nodes[0]
@@ -219,7 +232,7 @@ class YangHandler: 


                out_parts.append(name + ''.join(preds))

            else:

                local_part = part.split(':', 1)[1] if ':' in part else part

                local_part = part #.split(':', 1)[1] if ':' in part else part

                schema_path = schema_path + '/' + local_part if schema_path else '/' + local_part

                out_parts.append(part)

src/common/tools/rest_conf/server/restconf_server/YangModelDiscoverer.py

+42 −10
Original line number Diff line number Diff line
@@ -32,8 +32,14 @@ IMPORT_BLOCK_RE = re.compile(r"\bimport\s+([A-Za-z0-9_.-]+)\s*\{", re.IGNORECASE 
# import foo;  (very rare, but we’ll support it)

IMPORT_SEMI_RE  = re.compile(r"\bimport\s+([A-Za-z0-9_.-]+)\s*;", re.IGNORECASE)



# include foo { ... }  (most common form)

INCLUDE_BLOCK_RE = re.compile(r"\binclude\s+([A-Za-z0-9_.-]+)\s*\{", re.IGNORECASE)



def _parse_yang_file(path: Path) -> Tuple[Optional[str], Set[str]]:

# include foo;  (very rare, but we’ll support it)

INCLUDE_SEMI_RE  = re.compile(r"\binclude\s+([A-Za-z0-9_.-]+)\s*;", re.IGNORECASE)





def _parse_yang_file(path: Path) -> Tuple[Optional[str], Set[str], Set[str]]:

    path_stem = path.stem # file name without extension

    expected_module_name = path_stem.split('@', 1)[0]
@@ -54,14 +60,20 @@ def _parse_yang_file(path: Path) -> Tuple[Optional[str], Set[str]]: 
        raise Exception(MSG.format(str(module_name), str(expected_module_name)))



    module_imports = set()

    module_includes = set()

    if module_name is not None:

        module_imports.update(IMPORT_BLOCK_RE.findall(data))

        module_imports.update(IMPORT_SEMI_RE.findall(data))

        module_includes.update(INCLUDE_BLOCK_RE.findall(data))

        module_includes.update(INCLUDE_SEMI_RE.findall(data))



    # ignore modules importing themselves, just in case

    module_imports.discard(module_name)



    return module_name, module_imports

    # ignore modules including themselves, just in case

    module_includes.discard(module_name)



    return module_name, module_imports, module_includes





class YangModuleDiscoverer:
@@ -70,9 +82,9 @@ class YangModuleDiscoverer: 


        self._module_to_paths : Dict[str, List[Path]] = defaultdict(list)

        self._module_to_imports : Dict[str, Set[str]] = defaultdict(set)

        self._module_to_includes : Dict[str, Set[str]] = defaultdict(set)

        self._ordered_module_names : Optional[List[str]] = None





    def run(

        self, do_print_order : bool = False, do_log_order : bool = False,

        logger : Optional[logging.Logger] = None, level : int = logging.INFO
@@ -97,10 +109,30 @@ class YangModuleDiscoverer: 
            raise Exception(MSG.format(str(self._yang_search_path)))



        for yang_path in yang_root.rglob('*.yang'):

            module_name, module_imports = _parse_yang_file(yang_path)

            module_name, module_imports, module_includes = _parse_yang_file(yang_path)

            if module_name is None: continue

            self._module_to_paths[module_name].append(yang_path)

            self._module_to_imports[module_name] = module_imports

            self._module_to_paths.setdefault(module_name, list()).append(yang_path)

            self._module_to_imports.setdefault(module_name, set()).update(module_imports)

            self._module_to_includes.setdefault(module_name, set()).update(module_includes)



        # Propagate modules imported by included modules to modules including them:

        #   openconfig-platform includes openconfig-platform-common

        #   openconfig-platform-common imports (

        #       openconfig-platform-types, openconfig-extensions, openconfig-types

        #   )

        #   => propagate (

        #       openconfig-platform-types, openconfig-extensions, openconfig-types

        #   ) as imports of openconfig-platform

        #   => remove openconfig-platform-common from list of modules_to_imports as

        #      cannot be imported by itself

        included_modules : Set[str] = set()

        for module_name, module_includes in self._module_to_includes.items():

            for inc_mdl_name in module_includes:

                included_module_imports = self._module_to_imports.get(inc_mdl_name, set())

                self._module_to_imports.setdefault(module_name, set()).update(included_module_imports)

            included_modules.update(module_includes)

        for included_module in included_modules:

            self._module_to_imports.pop(included_module)



        if len(self._module_to_paths) == 0:

            MSG = 'No modules found in Path({:s})'
@@ -128,8 +160,8 @@ class YangModuleDiscoverer: 
    def _check_missing_modules(self) -> None:

        local_module_names = set(self._module_to_imports.keys())

        missing_modules : List[str] = list()

        for module_name, imported_modules in self._module_to_imports.items():

            missing = imported_modules.difference(local_module_names)

        for module_name, module_imports in self._module_to_imports.items():

            missing = module_imports.difference(local_module_names)

            if len(missing) == 0: continue

            missing_modules.append(

                '  {:s} => {:s}'.format(module_name, str(missing))
@@ -143,8 +175,8 @@ class YangModuleDiscoverer: 


    def _sort_modules(self) -> None:

        ts = TopologicalSorter()

        for module_name, imported_modules in self._module_to_imports.items():

            ts.add(module_name, *imported_modules)

        for module_name, module_imports in self._module_to_imports.items():

            ts.add(module_name, *module_imports)



        try:

            self._ordered_module_names = list(ts.static_order())   # raises CycleError on cycles