Commit cfc7927e authored by Pedro Duarte's avatar Pedro Duarte
Browse files

fix failed resource parsing

parent a6c95841
Loading
Loading
Loading
Loading
+176 −17
Original line number Diff line number Diff line
@@ -194,6 +194,104 @@ class GnmiSessionHandler:

        return result

    def _sanitize_unknown_identityrefs(self, obj: Any) -> int:
        """Recursively remove leaves that look like unknown identityrefs (ALLCAPS without module prefix)."""
        removed = 0
        if isinstance(obj, dict):
            keys_to_delete = []
            for k, v in obj.items():
                if isinstance(v, dict) or isinstance(v, list):
                    removed += self._sanitize_unknown_identityrefs(v)
                else:
                    if isinstance(v, str) and (':' not in v) and v.upper() == v and any(c.isalpha() for c in v):
                        keys_to_delete.append(k)
            for k in keys_to_delete:
                del obj[k]
                removed += 1
        elif isinstance(obj, list):
            for item in obj:
                removed += self._sanitize_unknown_identityrefs(item)
        return removed

    def _coerce_enum_like_booleans(self, obj: Any) -> int:
        """Recursively coerce boolean leaves to strings to avoid non-string enum errors; keep known boolean 'enabled'."""
        changed = 0
        if isinstance(obj, dict):
            for k, v in list(obj.items()):
                if isinstance(v, dict) or isinstance(v, list):
                    changed += self._coerce_enum_like_booleans(v)
                else:
                    if isinstance(v, bool):
                        if k in {'enabled'}:
                            continue
                        # Common patterns for status-like leaves
                        if 'status' in k or 'state' == k:
                            obj[k] = 'UP' if v else 'DOWN'
                        else:
                            obj[k] = 'true' if v else 'false'
                        changed += 1
        elif isinstance(obj, list):
            for item in obj:
                changed += self._coerce_enum_like_booleans(item)
        return changed

    def _enrich_missing_types(self, resource_path: str, data: Dict) -> int:
        """Populate default type identityrefs if missing to enable handlers to produce entries."""
        added = 0
        try:
            # Interfaces
            if resource_path.endswith('/interfaces'):
                interfaces_list = data.get('interface') or []
                if isinstance(interfaces_list, list):
                    for iface in interfaces_list:
                        state = iface.setdefault('state', {})
                        if 'type' not in state or not isinstance(state.get('type'), str) or len(state.get('type')) == 0:
                            state['type'] = 'iana-if-type:ethernetCsmacd'
                            added += 1
            # Components
            if resource_path.endswith('/components'):
                components_list = data.get('component') or []
                if isinstance(components_list, list):
                    for comp in components_list:
                        state = comp.setdefault('state', {})
                        if 'type' not in state or not isinstance(state.get('type'), str) or len(state.get('type')) == 0:
                            # Heuristic: if transceiver exists or name doesn't look like controller/node
                            has_transceiver = ('openconfig-platform-transceiver:transceiver' in comp) or ('transceiver' in comp)
                            name = comp.get('name', '')
                            if has_transceiver or name.startswith('veth') or name.endswith('-port'):
                                state['type'] = 'openconfig-platform-types:PORT'
                                added += 1
        except Exception: # defensive
            pass
        return added

    def _try_parse_with_sanitization(self, resource_path: str, data: Dict) -> List[Tuple[str, Dict[str, Any]]]:
        """Try parsing; on failure, sanitize generically (remove unknown identityrefs, coerce enum-like booleans) and retry."""
        try:
            return parse(resource_path, data, self._yang_handler)
        except Exception as e:
            self._logger.warning('Initial parse failed for %s: %s. Trying enrichment...', resource_path, e)
            # Enrich defaults
            enriched = json.loads(json.dumps(data))
            added_defaults = self._enrich_missing_types(resource_path, enriched)
            try:
                result = parse(resource_path, enriched, self._yang_handler)
                self._logger.info('Parse succeeded for %s after enrichment (defaults_added=%d).', resource_path, added_defaults)
                return result
            except Exception as e_enrich:
                self._logger.warning('Parse failed for %s after enrichment: %s. Applying generic sanitization...', resource_path, e_enrich)
                # Deep copy and sanitize
                sanitized = json.loads(json.dumps(enriched))
                removed = self._sanitize_unknown_identityrefs(sanitized)
                changed = self._coerce_enum_like_booleans(sanitized)
                added2 = self._enrich_missing_types(resource_path, sanitized)
                self._logger.info('Sanitization changes for %s: removed_unknown_identityrefs=%d, coerced_booleans=%d, defaults_added=%d', resource_path, removed, changed, added2)
                try:
                    return parse(resource_path, sanitized, self._yang_handler)
                except Exception as e2:
                    self._logger.error('Sanitized parse still failed for %s: %s. Skipping this batch.', resource_path, e2)
                    return []

    def _prefix_augmented_node(self, node_name: str) -> str:
        """
        Add module prefixes for augmented nodes so libyang can validate.
@@ -241,6 +339,19 @@ class GnmiSessionHandler:
            if segments[i] == 'config' and segments[i + 1] == 'health-indicator':
                self._logger.debug('Skipping non-OpenConfig leaf health-indicator at: %s', relative_path)
                return
        # - components/.../integrated-circuit/* (skip to avoid schema mismatch; not used by current handler)
        if any(seg == 'integrated-circuit' for seg in segments):
            self._logger.debug('Skipping integrated-circuit subtree: %s', relative_path)
            return

        # Detect top-level kind for later transforms
        top_kind = 'unknown'
        if len(segments) > 0:
            first_seg = segments[0]
            if first_seg.startswith('interface['):
                top_kind = 'interfaces'
            elif first_seg.startswith('component['):
                top_kind = 'components'

        cursor = root

@@ -293,6 +404,23 @@ class GnmiSessionHandler:
            node_name = self._prefix_augmented_node(segment)

            if is_last:
                # Coerce enumerations for known leaves (interfaces)
                if node_name in {'oper-status', 'admin-status'}:
                    # Accept bool or strings like 'true'/'false'/'0'/'1'
                    if isinstance(value, bool):
                        value = 'UP' if value else 'DOWN'
                    elif isinstance(value, str):
                        vlow = value.strip().lower()
                        if vlow in {'true', '1', 'up'}:
                            value = 'UP'
                        elif vlow in {'false', '0', 'down'}:
                            value = 'DOWN'
                # Prefix identityref values for type leaves
                if node_name == 'type' and i >= 1 and segments[i - 1] == 'state' and isinstance(value, str) and ':' not in value:
                    if top_kind == 'interfaces':
                        value = f'iana-if-type:{value}'
                    elif top_kind == 'components':
                        value = f'openconfig-platform-types:{value}'
                cursor[node_name] = value
            else:
                if node_name not in cursor or not isinstance(cursor[node_name], dict):
@@ -319,7 +447,23 @@ class GnmiSessionHandler:
        get_request.type = GetRequest.DataType.ALL
        get_request.encoding = Encoding.Value(self._encoding) if self._encoding else Encoding.JSON_IETF
        #get_request.use_models.add() # kept empty: return for all models supported
        supported_models = self._capabilities.get('supported_models', set())
        def _expand_query_paths(canonical_path: str) -> List[str]:
            # Remove namespace on first segment for initial try
            segs = canonical_path.strip('/').split('/')
            if not segs:
                return [canonical_path]
            first = segs[0]
            if ':' in first:
                first = first.split(':', 1)[1]
            base = first
            # Generic narrow queries to avoid unsupported subtrees
            if base == 'components':
                return ['/components/component/state', '/components/component/transceiver/state']
            if base == 'interfaces':
                return ['/interfaces/interface/state', '/interfaces/interface/config']
            # Default: non-namespaced canonical
            return ['/' + '/'.join([first] + segs[1:])]

        
        for i,resource_key in enumerate(resource_keys):
            str_resource_name = 'resource_key[#{:d}]'.format(i)
@@ -328,15 +472,10 @@ class GnmiSessionHandler:
                self._logger.debug('[GnmiSessionHandler:get] resource_key = {:s}'.format(str(resource_key)))
                str_path = get_path(resource_key)
                self._logger.debug('[GnmiSessionHandler:get] str_path = {:s}'.format(str(str_path)))
                # Extract namespace from path (e.g., "openconfig-platform" from "/openconfig-platform:components")
                parent_segment = str_path.strip('/').split('/')[0]
                parent_namespace = parent_segment.split(':')[0] if ':' in parent_segment else None
                if parent_namespace not in supported_models:
                    self._logger.warning('Skipping path %s because model %s is not advertised', str_path, parent_namespace)
                    continue
                
                self._logger.debug('Adding path to GET request: %s', str_path)
                get_request.path.append(path_from_string(str_path))
                expanded_paths = _expand_query_paths(str_path)
                self._logger.debug('Expanded GET paths for %s: %s', str_path, expanded_paths)
                for p in expanded_paths:
                    get_request.path.append(path_from_string(p))
            except Exception as e: # pylint: disable=broad-except
                MSG = 'Exception parsing {:s}: {:s}'
                self._logger.exception(MSG.format(str_resource_name, str(resource_key)))
@@ -357,20 +496,40 @@ class GnmiSessionHandler:
        metadata = [('username', self._username), ('password', self._password)]
        timeout = None # GNMI_SUBSCRIPTION_TIMEOUT = int(sampling_duration)
        
        # Try the original request first
        # Try the non-namespaced, narrow request first
        try:
            get_reply = self._stub.Get(get_request, metadata=metadata, timeout=timeout)
        except Exception as e:
            self._logger.warning('Get request failed with original paths: %s', e)
            self._logger.info('Retrying with non-namespaced paths...')
            self._logger.info('Retrying with namespaced equivalents...')

            fallback_request = GetRequest()
            fallback_request.type = GetRequest.DataType.ALL
            fallback_request.encoding = get_request.encoding

            for path in get_request.path:
                path_str = path_to_string(path)
                fallback_request.path.append(path_from_string(path_str.split(':', 1)[1] if ':' in path_str else path_str))
            # Re-add namespace of canonical path on first segment if present
            for rk in resource_keys:
                try:
                    canonical = get_path(rk)
                    segs = canonical.strip('/').split('/')
                    if not segs:
                        continue
                    first = segs[0]
                    ns = first.split(':', 1)[0] if ':' in first else None
                    if not ns:
                        continue
                    base = first.split(':', 1)[1] if ':' in first else first
                    # Rebuild namespaced narrow paths
                    if base == 'components':
                        ns_paths = [f'/{ns}:components/component/state', f'/{ns}:components/component/transceiver/state']
                    elif base == 'interfaces':
                        ns_paths = [f'/{ns}:interfaces/interface/state', f'/{ns}:interfaces/interface/config']
                    else:
                        ns_paths = [f'/{ns}:{"/".join([base]+segs[1:])}']
                    for p in ns_paths:
                        fallback_request.path.append(path_from_string(p))
                except Exception:
                    continue
            
            get_reply = self._stub.Get(fallback_request, metadata=metadata, timeout=timeout)
        
@@ -413,8 +572,8 @@ class GnmiSessionHandler:
                        reconstructed_data = self._reconstruct_object_from_updates(notification.update)
                        self._logger.debug('Reconstructed data keys: %s', list(reconstructed_data.keys()))
                        self._logger.debug('Calling parse with path: %s and data: %s', common_path, reconstructed_data)
                        # Parse the reconstructed object at the resource level
                        results.extend(parse(common_path, reconstructed_data, self._yang_handler))
                        # Parse the reconstructed object at the resource level with generic sanitization fallback
                        results.extend(self._try_parse_with_sanitization(common_path, reconstructed_data))
                    except Exception as e:
                        self._logger.exception('Exception processing notification for path %s', common_path)
                        results.append((common_path, e))
+5 −5
Original line number Diff line number Diff line
@@ -113,7 +113,7 @@ class InterfaceHandler(_Handler):
            #yang_interface.merge_data_dict(interface, strict=True, validate=False)

            interface_state = interface.get('state', {})
            interface_type = interface_state.get('type')
            interface_type = interface_state.get('type', 'ethernetCsmacd')
            if interface_type is None: continue
            interface_type = interface_type.split(':')[-1]
            if interface_type not in {'ethernetCsmacd'}: continue
@@ -121,7 +121,7 @@ class InterfaceHandler(_Handler):
            _interface = {
                'name'         : interface_name,
                'type'         : interface_type,
                'mtu'          : interface_state['mtu'],
                'mtu'          : interface_state.get('mtu'),
                'admin-status' : interface_state['admin-status'],
                'oper-status'  : interface_state['oper-status'],
                'management'   : interface_state['management'],
@@ -145,9 +145,9 @@ class InterfaceHandler(_Handler):

                _ethernet = {
                    'mac-address'           : ethernet_state['mac-address'],
                    'hw-mac-address'        : ethernet_state['hw-mac-address'],
                    'port-speed'            : ethernet_state['port-speed'].split(':')[-1],
                    'negotiated-port-speed' : ethernet_state['negotiated-port-speed'].split(':')[-1],
                    'hw-mac-address'        : ethernet_state.get('hw-mac-address'),
                    'port-speed'            : ethernet_state['port-speed'].split('_')[-1],
                    'negotiated-port-speed' : ethernet_state['negotiated-port-speed'].split('_')[-1],
                }
                entry_ethernet_key = '{:s}/ethernet'.format(entry_interface_key)
                entries.append((entry_ethernet_key, _ethernet))
+15 −15
Original line number Diff line number Diff line
@@ -40,21 +40,21 @@ class InterfaceCounterHandler(_Handler):
            interface_counters = interface.get('state', {}).get('counters', {})
            _interface = {
                'name'              : interface_name,
                'in-broadcast-pkts' : interface_counters['in_broadcast_pkts' ],
                'in-discards'       : interface_counters['in_discards'       ],
                'in-errors'         : interface_counters['in_errors'         ],
                'in-fcs-errors'     : interface_counters['in_fcs_errors'     ],
                'in-multicast-pkts' : interface_counters['in_multicast_pkts' ],
                'in-octets'         : interface_counters['in_octets'         ],
                'in-pkts'           : interface_counters['in_pkts'           ],
                'in-unicast-pkts'   : interface_counters['in_unicast_pkts'   ],
                'out-broadcast-pkts': interface_counters['out_broadcast_pkts'],
                'out-discards'      : interface_counters['out_discards'      ],
                'out-errors'        : interface_counters['out_errors'        ],
                'out-multicast-pkts': interface_counters['out_multicast_pkts'],
                'out-octets'        : interface_counters['out_octets'        ],
                'out-pkts'          : interface_counters['out_pkts'          ],
                'out-unicast-pkts'  : interface_counters['out_unicast_pkts'  ],
                'in-broadcast-pkts' : interface_counters['in-broadcast-pkts' ],
                'in-discards'       : interface_counters['in-discards'       ],
                'in-errors'         : interface_counters['in-errors'         ],
                'in-fcs-errors'     : interface_counters['in-fcs-errors'     ],
                'in-multicast-pkts' : interface_counters['in-multicast-pkts' ],
                'in-octets'         : interface_counters['in-octets'         ],
                'in-pkts'           : interface_counters['in-pkts'           ],
                'in-unicast-pkts'   : interface_counters['in-unicast-pkts'   ],
                'out-broadcast-pkts': interface_counters['out-broadcast-pkts'],
                'out-discards'      : interface_counters['out-discards'      ],
                'out-errors'        : interface_counters['out-errors'        ],
                'out-multicast-pkts': interface_counters['out-multicast-pkts'],
                'out-octets'        : interface_counters['out-octets'        ],
                'out-pkts'          : interface_counters['out-pkts'          ],
                'out-unicast-pkts'  : interface_counters['out-unicast-pkts'  ],
            }
            LOGGER.debug('interface = {:s}'.format(str(interface)))