High Memory Usage per Chassis and Blade-radware-alteon-os

error
health-checks
alteon-os
radware
High Memory Usage per Chassis and Blade-radware-alteon-os
0

#1

High Memory Usage per Chassis and Blade-radware-alteon-os

Vendor: radware

OS: alteon-os

Description:
Alert when Memory usage is high

Remediation Steps:
Review the load on this thread to see if the memory utilization is valid.

How does this work?
This script runs the “/config/mpMemStatsTotal” and “/config/mpMemStatsFree” to check the % utilization which is necessary for Indeni to determine if the Radware device has high memory utilization.

Why is this important?
Memory utilization from the Management Process is key. If the management process begins to peg the available memory heavily, the admin should be made aware as this is indicative of a lack of resources available. Determining total allocated for the device is crucial for determining threshold levels.

Without Indeni how would you find this?
An administrator would need to log in to the device and run a CLI command or run the API commands.

alteon-api-config-SpMemUseStatsTable

#! META
name: alteon-api-config-SpMemUseStatsTable
description: get the total size of the Switch Processor RAM 
type: monitoring
monitoring_interval: 1 minute
includes_resource_data: true
requires:
    os.name: "alteon-os"
    vendor: "radware"
    or:
        -
            vsx: "true"
        -
            standalone: "true"

#! COMMENTS
memory-free-kbytes:
    skip-documentation: true
memory-total-kbytes:
    skip-documentation: true
memory-usage:
    why: |
        Memory utilization from the Management Process is key. If the management process begins to peg the available memory heavily, the admin should be made aware as this is indicative of a lack of resources available. Determining total allocated for the device is crucial for determining threshold levels.
    how: |
        This script runs the "/config/mpMemStatsTotal" and "/config/mpMemStatsFree" to check the % utilization which is necessary for Indeni to determine if the Radware device has high memory utilization.
    without-indeni: |
        An administrator would need to log in to the device and run a CLI command or run the API commands.
    can-with-snmp: true
    can-with-syslog: false
    vendor-provided-management: |
        Can be done through Management GUI (Vision or Alteon VX).

#! REMOTE::HTTP
url: /config/SpMemUseStatsTable?&props=Index,InitSizeTo1Margin,CurProcSize
protocol: HTTPS

#! PARSER::JSON

_metrics:
    -
        _groups:
            "$.SpMemUseStatsTable[0:]":    
                _temp:
                    "Index":
                        _value: Index
                    "spMemCurrent":
                        _value: CurProcSize
                    "spMemTotal":
                        _value: InitSizeTo1Margin
                _tags:
                    "im.name":
                        _constant: "memory-usage"
                    "live-config":
                        _constant: "true"
                    "display-name":
                        _constant: "Memory - Used"
                    "im.dstype.displayType":
                        _constant: "percentage"
                    "im.identity-tags":
                        _constant: "name"
                    "resource-metric":
                        _constant: "true"
        _transform:
            _tags:
                "name" : |
                    {
                        print "Switch Processor " temp("Index")
                    }
            _value.double: | 
                { 
                    print ((temp("spMemCurrent") / temp("spMemTotal")) * 100)
                }

high_per_chassis_blade_memory_usage

package com.indeni.server.rules.library

import com.indeni.ruleengine.Scope.{Scope, ScopeValueHelper}
import com.indeni.ruleengine.expressions.Expression
import com.indeni.ruleengine.expressions.conditions.GreaterThanOrEqual
import com.indeni.ruleengine.expressions.core.{StatusTreeExpression, _}
import com.indeni.ruleengine.expressions.data.{SelectTagsExpression, SelectTimeSeriesExpression, TimeSeriesExpression}
import com.indeni.ruleengine.expressions.math.AverageExpression
import com.indeni.ruleengine.expressions.scope.ScopableExpression
import com.indeni.server.common.ParameterValue
import com.indeni.server.common.data.conditions.{Equals, True}
import com.indeni.server.params.ParameterDefinition
import com.indeni.server.params.ParameterDefinition.UIType
import com.indeni.server.rules._
import com.indeni.server.rules.config.expressions.DynamicParameterExpression
import com.indeni.server.rules.library.core.PerDeviceRule
import com.indeni.server.sensor.models.managementprocess.alerts.dto.AlertSeverity


/**
  * Created by amir on 04/02/2016.
  */
case class HighPerChassisBladeMemoryUsageRule(context: RuleContext) extends PerDeviceRule {

  private val highThresholdParameterName: String = "High_Threshold_of_Memory_Usage"

  private val highThresholdParameter = new ParameterDefinition(highThresholdParameterName,
    "",
    "High Threshold of Memory Usage",
    "What is the threshold for the memory usage for which once it is crossed an issue will be triggerd.",
    UIType.DOUBLE,
    new ParameterValue((85.0).asInstanceOf[Object])
  )

  override val metadata: RuleMetadata = RuleMetadata.builder("high_per_chassis_blade_memory_usage", "High Memory Usage per Chassis and Blade", "Alert when Memory usage is high", AlertSeverity.ERROR).configParameter(highThresholdParameter).build()

  override def expressionTree: StatusTreeExpression = {

    val usagePercentage = AverageExpression(TimeSeriesExpression[Double]("memory-usage"))
    val usagePercentageThreshold = DynamicParameterExpression.withConstantDefault(highThresholdParameter.getName, highThresholdParameter.getDefaultValue.asDouble.toDouble).noneable
    val isUsagePercentageAboveThreshold = GreaterThanOrEqual(usagePercentage, usagePercentageThreshold)

    val mountSpaceFailDescription = new ScopableExpression[String] {
      override protected def evalWithScope(time: Long, scope: Scope): String =
        "Memory usage (" + usagePercentage.eval(time) + "%) above threshold (" + usagePercentageThreshold.eval(time) + "%) " +
          "for chassis: " + scope.getVisible("Chassis").get + ", blade: " + scope.getVisible("Blade").get

      override def args: Set[Expression[_]] = Set(usagePercentage, usagePercentageThreshold)
    }
    val mountSpaceFailHeadline = new ScopableExpression[String] {
      override protected def evalWithScope(time: Long, scope: Scope): String = "chassis: " + scope.getVisible("Chassis").get + ", blade: " + scope.getVisible("Blade").get

      override def args: Set[Expression[_]] = Set()
    }
    val tsQuery = SelectTimeSeriesExpression[Double](context.tsDao, Set("memory-usage"), denseOnly = false)
    val forTsCondition = StatusTreeExpression(tsQuery, isUsagePercentageAboveThreshold).withSecondaryInfo(
      mountSpaceFailHeadline, mountSpaceFailDescription, title = "Problematic Blades"
    ).asCondition()

    val chassisBladeQuery = SelectTagsExpression(context.tsDao, Set("Chassis", "Blade"), True)
    val highMemoryUsagePerDevicePerChassisBladeLogic = StatusTreeExpression(chassisBladeQuery, forTsCondition)
      .withoutInfo().asCondition()

    val headline = ConstantExpression("High memory usage")
    val description = ConstantExpression("The memory usage in the operating system is higher than the high threshold.")
    val remediation = ConditionalRemediationSteps("Review the load on this thread to see if the memory utilization is valid.",
      ConditionalRemediationSteps.OS_NXOS ->
        """1. Check from the Indeni  the memory utilization history graph for this device an review the pattern. Correlate any change to the pattern with any configuration change.
          |2. The next NX-OS commands output can inform whether the platform memory utilization is normal or un-expected:
          | a. "show system resources"
          | b. "show processes memory"
          |3. For more information, please review: <a target="_blank" href="http://docwiki.cisco.com/wiki/Cisco_Nexus_7000_Series_NX-OS_Troubleshooting_Guide_--_Troubleshooting_Memory">Troubleshooting Guide For High Memory Utilization</a>.""".stripMargin,
      ConditionalRemediationSteps.VENDOR_JUNIPER ->
        """|1. On the device command line interface execute "show chassis routing-engine" command to check overall routing engine memory usage.
           |2. Run "show system processes extensive" command to review the memory allocation status for processes.
           |3. Identify the processes which are consuming too much memory.
           |4. Consider turning off some processes which are not vital to ensure bigger memory space allocation for other sessions and processes.
           |5. Review the following article on Juniper tech support site: <a target="_blank" href="https://www.juniper.net/documentation/en_US/release-independent/nce/topics/task/operational/security-policy-memory-testing.html">Checking Memory Status</a>.""".stripMargin
    )

    val devicesFilter = Equals("model", "CheckPoint61k")
    val devicesQuery = SelectTagsExpression(context.metaDao, Set(DeviceKey), devicesFilter)

    StatusTreeExpression(devicesQuery, highMemoryUsagePerDevicePerChassisBladeLogic).withRootInfo(
      headline, description, remediation
    )
  }
}