Hardware and software clock mismatch-linux-False

error
health-checks
false
linux
Hardware and software clock mismatch-linux-False
0

#1

Hardware and software clock mismatch-linux-False

Vendor: linux

OS: False

Description:
indeni will trigger an issue when a hardware and software clock are different on a device.

Remediation Steps:
Read http://www.tldp.org/LDP/sag/html/hw-sw-clocks.html on why this is important.

How does this work?
The time from both the hardware clock and system clock is listed, and the difference is calculated.

Why is this important?
Each device has a hardware clock (RTC) and a system clock. If the hardware clock drifts too far away from the system clock, this can mean that the clock will be incorrect after a reboot.

Without Indeni how would you find this?
An administrator could login and manually run the command.

linux-compare-hwclock

#! META
name: linux-compare-hwclock
description: Compare OS clock with HW clock.
type: monitoring
monitoring_interval: 5 minutes
requires:
    or:
        -
            linux-based: true
        -
            linux-busybox: true
    this_tag_disables_this_script: this_is_intentional

#! COMMENTS
hw-clock-difference:
    why: |
        Each device has a hardware clock (RTC) and a system clock. If the hardware clock drifts too far away from the system clock, this can mean that the clock will be incorrect after a reboot.
    how: |
        The time from both the hardware clock and system clock is listed, and the difference is calculated.
    without-indeni: |
        An administrator could login and manually run the command.
    can-with-snmp: false
    can-with-syslog: false
    vendor-provided-management: |
        This is only accessible from the command line interface.

#! REMOTE::SSH
i=0; while [ $i -lt 5 ]; do echo -n "hwclock " && ${nice-path} -n 15 hwclock --show && echo -n "osclock " && ${nice-path} -n 15 clock; i=$[$i+1]; done

#! PARSER::AWK

############
# Script explanation: Sometimes the result of the command is very wrong. This each command is run several times, and there is a vote on which result is correct.
# We will run each command, and a result needs three votes to win.
###########

function notNegative(input) {
	# We dont want a negative value
	if ( input < 0) {
		input = -input
	}
	return input
}

function election (clockEpoch) {

	# Election
	for (id in clockEpoch) {
		for (id2 in clockEpoch) {
		
			# Check the difference between
			diff = clockEpoch[id] - clockEpoch[id2]
			
			# Remove any minus signs
			diff = notNegative(diff)
			
			# If the two commands do not differ more than 10 seconds, then its ok
			if (diff <= 10) {
				vote[id]++
			}
		}
	}
	
	# Check vote result
	for (id in vote) {
		if (highscore) {
			if (highscore < vote[id]) {
				highscore = vote[id]
				winner = id
			}
		} else {
			highscore = vote[id]
			winner = id
		}
	}
	
	# the candidate need more then voteMajority nr of votes, otherwise the result is not trustworthy.
	if (vote[winner] < voteMajority) {
		winner = ""
	}
	highscore = ""
	delete vote
	
	return winner
}


#Checkpoint format
#hwclock Thu Jan  24 13:14:41 2017  -0.015839 seconds
#F5 format
#hwclock Wed 18 Jan 2017 09:32:14 PM PST  -0.000999 seconds

/^hwclock/ {

	if(match($0,/hwclock\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[0-9]{1,2}/)){
		#Checkpoint format
		iHardwareClock++

		hwYear = $6
		hwMonth = parseMonthThreeLetter($3)
		hwDay = $4
		split($5, timeArr, ":")
		hwHour = timeArr[1]
		hwMinute = timeArr[2]
		hwSecond = timeArr[3]

		hwClockEpoch[iHardwareClock] = datetime(hwYear,hwMonth,hwDay,hwHour,hwMinute,hwSecond)
		
	} else if(match($0,/hwclock\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+[0-9]{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-9]{4}/)){
		#F5 format
		iHardwareClock++

		hwYear = $5
		hwMonth = parseMonthThreeLetter($4)
		hwDay = $3
		split($6, timeArr, ":")
		hwHour = timeArr[1]
		hwMinute = timeArr[2]
		hwSecond = timeArr[3]

		hwClockEpoch[iHardwareClock] = datetime(hwYear,hwMonth,hwDay,hwHour,hwMinute,hwSecond)

	}
}

#Checkpoint format:
#osclock Thu Jan  5 13:14:48 2017  -1.000184 seconds
#F5 format
#osclock Wed 18 Jan 2017 09:32:14 PM PST  -0.001144 seconds

/^osclock/ {
	
	if(match($0,/^osclock\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+[0-9]{1,2}/)){
		#Checkpoint format
		iOsClock++
		
		osYear = $6
		osMonth = parseMonthThreeLetter($3)
		osDay = $4
		split($5, timeArr, ":")
		osHour = timeArr[1]
		osMinute = timeArr[2]
		osSecond = timeArr[3]
		
		osClockEpoch[iOsClock] = datetime(osYear, osMonth, osDay, osHour, osMinute, osSecond)
	
	} else if(match($0,/^osclock\s+(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s+[0-9]{1,2}\s+(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s[0-9]{4}/)){
		#F5 format
		iOsClock++

		osYear = $5
		osMonth = parseMonthThreeLetter($4)
		osDay = $3
		split($6, timeArr, ":")
		osHour = timeArr[1]
		osMinute = timeArr[2]
		osSecond = timeArr[3]

		osClockEpoch[iOsClock] = datetime(osYear, osMonth, osDay, osHour, osMinute, osSecond)
	}
}

END {
	
	#Make sure that we actually got results in the arrays
	if(iOsClock > 0 && iHardwareClock > 0){

		# Lets hold two elections, one for osclock and one for hw clock.
		# Each of the 5 results will be compared to see if it is within 10 seconds from the other 4 results.
		# If one result is within this time with three other results, then it is declared a winner.
		
		voteMajority = 3
		
		## HW clock
		winnerHwClock = election(hwClockEpoch)
		
		
		## OS clock
		winnerOsClock = election(osClockEpoch)

		# Make sure we have a winner from each side before we write the metric
		if (winnerOsClock && winnerHwClock) {
			difference = hwClockEpoch[winnerHwClock] - osClockEpoch[winnerOsClock]

			difference = notNegative(difference)
			
			# There are still times where the clock difference is incorrect by a very large number. To prevent this it will not be allowed for the difference to be more than 10 million seconds, which is ~115 days.
			if (difference >! "10000000") {
				writeDoubleMetric("hw-clock-difference", null, "gauge", "600", difference)
			}
		}
	}
}

cross_vendor_hw_sw_clock_diff

package com.indeni.server.rules.library

import com.indeni.ruleengine.expressions.conditions.GreaterThanOrEqual
import com.indeni.ruleengine.expressions.core._
import com.indeni.ruleengine.expressions.data.{SelectTagsExpression, SelectTimeSeriesExpression, TimeSeriesExpression}
import com.indeni.server.common.data.conditions.True
import com.indeni.server.rules._
import com.indeni.server.rules.library.core.PerDeviceRule
import com.indeni.server.sensor.models.managementprocess.alerts.dto.AlertSeverity

case class CompareHwSwClockRule() extends PerDeviceRule with RuleHelper {

  override val metadata: RuleMetadata = RuleMetadata.builder("cross_vendor_hw_sw_clock_diff", "All Devices: Hardware and software clock mismatch",
    "indeni will trigger an issue when a hardware and software clock are different on a device.", AlertSeverity.ERROR).build()

  override def expressionTree(context: RuleContext): StatusTreeExpression = {
    val actualValue = TimeSeriesExpression[Double]("hw-clock-difference").last

    StatusTreeExpression(
      // Which objects to pull (normally, devices)
      SelectTagsExpression(context.metaDao, Set(DeviceKey), True),

      StatusTreeExpression(
            // The time-series we check the test condition against:
            SelectTimeSeriesExpression[Double](context.tsDao, Set("hw-clock-difference"), denseOnly = false),

            // The condition which, if true, we have an issue. Checked against the time-series we've collected
            GreaterThanOrEqual(
              actualValue,
              ConstantExpression(Some(10.0)))

            // The Alert Item to add for this specific item
      ).withRootInfo(
          getHeadline(),
          scopableStringFormatExpression("The hardware and software clock differ by %.0f seconds. This situation may result in weird issues.", actualValue),
            ConditionalRemediationSteps("Read http://www.tldp.org/LDP/sag/html/hw-sw-clocks.html on why this is important.",
              ConditionalRemediationSteps.VENDOR_CP -> "https://forums.checkpoint.com/forums/message.jspa?messageID=50368"
          )
      ).asCondition()
    ).withoutInfo()
  }
}