Difference between revisions of "Monitoring openvz resources using munin"

From OpenVZ Virtuozzo Containers Wiki
Jump to: navigation, search
(Wikified (at least some more), added reference to another plugin, which uses vzlist)
(Undo revision 10871 by 217.195.167.74 (Talk))
 
(8 intermediate revisions by 7 users not shown)
Line 1: Line 1:
 
[[Category: Monitoring]]
 
[[Category: Monitoring]]
  
There are several plugins available on this page to monitor beancounter values in [http://munin.projects.linpro.no/ Munin].
+
[http://exchange.munin-monitoring.org/plugins/openvzcpu/details There is a plugin available on the Munin Exchange web site] which tracks CPU usage accross different containers.
The third one tries to combine the other two.
+
 
 +
[http://exchange.munin-monitoring.org/plugins/openvz_/details A second is also available via the Munin plugin exchange] which tracks beancounter values.
 +
 
 +
Additionally, there are several plugins available on this page to monitor beancounter values in [http://munin.projects.linpro.no/ Munin].
 +
The third one tries to combine the other two. (FIXME, maybe these should live in the Munin Exchange too?)
  
 
= "Simple" munin plugin =
 
= "Simple" munin plugin =
Line 195: Line 199:
  
 
= Extended plugin from Jan Tomasek =
 
= Extended plugin from Jan Tomasek =
Jan has posted another plugin on http://forum.openvz.org/index.php?t=msg&goto=15122, where I've fixed two things:
+
* Jan has posted another plugin on http://forum.openvz.org/index.php?t=msg&goto=15122, where I've fixed two things:
# "exit 0" in the "config" block
+
*# "exit 0" in the "config" block
# Replaced "vals=($str); echo ${vals[0]}" with "echo ${str%% *}" (the former was causing problems I don't remember anymore)
+
*# Replaced "vals=($str); echo ${vals[0]}" with "echo ${str%% *}" (the former was causing problems I don't remember anymore)
 
+
* v1.3.2 (2008/08/09)
 +
*# If only 1 variable is graphed, also display maxheld, barrier and limit
 
<source lang="bash">
 
<source lang="bash">
 
#!/bin/sh
 
#!/bin/sh
Line 214: Line 219:
 
#  - "exit 0" in "config" block
 
#  - "exit 0" in "config" block
 
#  - Use "echo ${str%% *}" in "suggest", instead of "vals=($str); echo ${vals[0]}"
 
#  - Use "echo ${str%% *}" in "suggest", instead of "vals=($str); echo ${vals[0]}"
 +
#
 +
# Revision 1.3.2 2008/08/09 12:30:00 Christian Rubbert <crubbert@xrc.de>
 +
# * Feature
 +
#  - If only 1 variable is graphed, also display maxheld, barrier and limit
 
#
 
#
 
# Original revision taken from:
 
# Original revision taken from:
Line 220: Line 229:
 
#%# family=auto
 
#%# family=auto
 
#%# capabilities=autoconf suggest
 
#%# capabilities=autoconf suggest
 
+
 
VEID=`basename $0 | sed -e 's/^vebc_.*_//'`;
 
VEID=`basename $0 | sed -e 's/^vebc_.*_//'`;
 
STATS=`basename $0 | sed -e 's/^vebc_//' -e 's/_[0-9]*$//' -e 's/_/ /g'`
 
STATS=`basename $0 | sed -e 's/^vebc_//' -e 's/_[0-9]*$//' -e 's/_/ /g'`
 
+
x=0; STATSCNT=`for i in $STATS; do x=$[$x+1]; done; echo $x`
 +
 
if [ "$1" = "autoconf" ]; then
 
if [ "$1" = "autoconf" ]; then
 
     if [ -r /proc/bc/0/resources ]; then
 
     if [ -r /proc/bc/0/resources ]; then
Line 233: Line 243:
 
     fi
 
     fi
 
fi
 
fi
 
+
 
if [ "$1" = "suggest" ]; then
 
if [ "$1" = "suggest" ]; then
 
     if [ -r /proc/bc/0/resources ]; then
 
     if [ -r /proc/bc/0/resources ]; then
Line 245: Line 255:
 
     fi
 
     fi
 
fi
 
fi
 
+
 
if [ ! -f /proc/bc/$VEID/resources ]; then
 
if [ ! -f /proc/bc/$VEID/resources ]; then
 
     exit 0;
 
     exit 0;
 
fi
 
fi
 
+
 
if [ "$1" = "config" ]; then
 
if [ "$1" = "config" ]; then
 
     #echo "graph_order down up"
 
     #echo "graph_order down up"
Line 255: Line 265:
 
     echo "graph_vlabel bean counters"
 
     echo "graph_vlabel bean counters"
 
     echo "graph_category VE$VEID"
 
     echo "graph_category VE$VEID"
 +
 
     # Note on URLs. General graph info is by munin version 1.2.5
 
     # Note on URLs. General graph info is by munin version 1.2.5
 
     # accepted even with HTML code. But for value.info it escapes URL,
 
     # accepted even with HTML code. But for value.info it escapes URL,
Line 261: Line 272:
 
     echo "graph_info VE bean counters info. Documentation of the OpenVZ resource management is located at <a href=\"
 
     echo "graph_info VE bean counters info. Documentation of the OpenVZ resource management is located at <a href=\"
 
http://wiki.openvz.org/UBC\">http://wiki.openvz.org/UBC</a>."
 
http://wiki.openvz.org/UBC\">http://wiki.openvz.org/UBC</a>."
 
+
 
     cat /proc/bc/$VEID/resources | while read name value top warn max stuff ; do
 
     cat /proc/bc/$VEID/resources | while read name value top warn max stuff ; do
 
         for statname in $STATS; do
 
         for statname in $STATS; do
Line 276: Line 287:
 
         done
 
         done
 
     done
 
     done
 +
 +
    if [ "$STATSCNT" = "1" ]; then
 +
        echo maxheld.label Maxheld
 +
        echo maxheld.draw LINE2
 +
        echo maxheld.info Maximum value
 +
        echo barrier.label Barrier
 +
        echo barrier.draw LINE2
 +
        echo barrier.info Barrier
 +
        echo limit.label Limit
 +
        echo limit.draw LINE2
 +
        echo limit.info Limit
 +
    fi
 
     exit 0
 
     exit 0
 
fi;
 
fi;
 
+
cat /proc/bc/$VEID/resources | while read name value stuff ; do
+
cat /proc/bc/$VEID/resources | while read name value top warn max stuff ; do
 
     for statname in $STATS; do
 
     for statname in $STATS; do
 
     if [ "$name" = "$statname" ]; then
 
     if [ "$name" = "$statname" ]; then
 
         echo $name".value "$value;
 
         echo $name".value "$value;
 +
 +
if [ "$STATSCNT" = "1" ]; then
 +
    echo maxheld.value $top
 +
    echo barrier.value $warn
 +
    echo limit.value $max
 +
fi
 
     fi
 
     fi
 
     done
 
     done
done
+
done</source>
</source>
 
 
 
  
 
= Munin plugin setup =
 
= Munin plugin setup =
Line 336: Line 363:
  
  
= Notes =
+
= Alternative: Using vzlist =
== Another plugin, using vzlist ==
+
This simple script will generate an overview of the requested stat from all VE's.
There is another plugin available at http://muninexchange.projects.linpro.no/?view&phid=249 - using vzlist and PHP.
+
When using vzlist your saved from any changes to the beancounters.
Especially using "vzlist" to only gather the requested fields might be a faster approach.
+
 
 +
== Usage ==
 +
Simply append the variable you want to monitor:
 +
<source lang="bash">ln -s /usr/share/munin/plugins/openvz_ /etc/munin/plugins/openvz_physpages</source>
 +
 
 +
<source lang="bash">
 +
#!/bin/sh
 +
#
 +
# Munin's plugin to monitor OpenVZ bean counters.
 +
#
 +
# $Log$
 +
# 2008/08/14 Rene Weselowski <http://www.dead.at>
 +
#
 +
#%# family=auto
 +
#%# capabilities=autoconf
 +
 
 +
ATTRIBUTE=`basename $0 | sed 's/^openvz_//g'`
 +
 
 +
 
 +
if [ "$1" = "autoconf" ]; then
 +
        echo yes
 +
        exit 0
 +
fi
 +
 
 +
if [ "$1" = "config" ]; then
 +
        echo "graph_title $ATTRIBUTE"
 +
        echo "graph_args --base 1000 -l 0"
 +
        echo "graph_scale yes"
 +
        echo "graph_vlabel $ATTRIBUTE Value"
 +
        echo "graph_category openvz"
 +
        echo "graph_info This graph shows OpenVZ: $ATTRIBUTE"
 +
        vzlist -a -H -o hostname | awk '{gsub(/\./,"_",$1)
 +
        print("'$ATTRIBUTE'"$1".label "$1"\n" \
 +
        "'$ATTRIBUTE'"$1".info '$ATTRIBUTE' for VE"$1)}'
 +
        exit 0
 +
fi
 +
 
 +
vzlist -a -H -o hostname,$ATTRIBUTE | awk '{gsub(/\./,"_",$1)
 +
        print("'$ATTRIBUTE'"$1".value "$2)}'
 +
 
 +
</source>

Latest revision as of 06:32, 31 August 2011


There is a plugin available on the Munin Exchange web site which tracks CPU usage accross different containers.

A second is also available via the Munin plugin exchange which tracks beancounter values.

Additionally, there are several plugins available on this page to monitor beancounter values in Munin. The third one tries to combine the other two. (FIXME, maybe these should live in the Munin Exchange too?)

"Simple" munin plugin[edit]

The plugin listed below grabs all the beancounters' values.

#!/bin/sh
#
# plugin to monitor OpenVZ bean counters.
#
#
#%# family=auto
#%# capabilities=autoconf suggest

ATTR=`basename $0 | sed 's/^vebc_//g'`

if [ "$1" = "autoconf" ]; then
        if [ -r /proc/bc/0/resources ]; then
                echo yes
                exit 0
        else
                echo "no (/proc/bc/0/resources not found)"
                exit 1
        fi
fi

if [ "$1" = "suggest" ]; then
        if [ -r /proc/bc/0/resources ]; then
            cat /proc/bc/0/resources |
            while read str; do
                vals=($str)
                echo ${vals[0]}
            done
                exit 0
        else
                exit 1
        fi
fi

if [ "$1" = "config" ]; then
#       echo "graph_order down up"
        echo "graph_title $ATTR beancounter for containers"
        echo 'graph_category system'
        echo "graph_info 'Containers beancounters info'"

        for CTID in `ls -d1 /proc/bc/???`; do
            id=`basename $CTID`
            grep $ATTR $CTID/resources |
            while read str; do

                vals=($str)
                name=${vals[0]}
                echo ${id}.label $id
                echo "${id}.warning  ${vals[3]}"
                echo "${id}.critical  ${vals[4]}"
            done
        done

        exit 0
fi;

for CTID in `ls -d1 /proc/bc/???`; do
    id=`basename $CTID`
    grep $ATTR $CTID/resources |
    while read str; do
        vals=($str)
        name=${vals[0]}
        echo "$id.value ${vals[1]}"
    done
done

Extended Version for old system using user_beancounter[edit]

Put it with the munin plugins and make a link for every graph which should be produced named like:

vebc_VALUENAME1_VALUENAME2_..._CTID

e.g.: vebc_numflock_numpty_numsiginfo_101

#!/bin/sh
#
# plugin to monitor OpenVZ bean counters.
#
#
#%# family=auto
#%# capabilities=autoconf suggest

ATTR=`basename $0 | sed -e 's/^vebc_.*_//'`
STATS=`basename $0 | sed -e 's/^vebc_//' -e 's/_[0-9]*$//' -e 's/_/ /g'`

if [ "$1" = "autoconf" ]; then
        if [ -r /proc/bc/0/resources ]; then
                echo yes
                exit 0
        else
                echo "no (/proc/bc/0/resources not found)"
                exit 1
        fi
fi

if [ "$1" = "suggest" ]; then
        if [ -r /proc/bc/0/resources ]; then
            cat /proc/bc/0/resources |
            while read str; do
                vals=($str)
                echo ${vals[0]}
            done
                exit 0
        else
                exit 1
        fi
fi

if [ "$1" = "config" ]; then
#       echo "graph_order down up"
        echo "graph_title beancounter for CT$ATTR: $STATS"
        echo "graph_category CT$ATTR"
        echo "graph_info 'Container bean counters info'"

        readme="false"
        cat /proc/user_beancounters | while read myid stuff; do
                line=""
                if [ "$myid" == "$ATTR:" ]; then
                        readme="true"
                        line="$stuff"
                        echo $line
                else
                        loid=`echo $myid | sed -e 's/.*:/:/'`
                        if [ "$loid" == ":" ]; then
                                readme="false"
                        fi
                        if [ "$readme" == "true" ]; then
                                line="$myid $stuff"
                        echo $line
                        fi
                fi
        done | while read name value top warn max; do
                okname="dummy"
                for statname in $STATS; do
                        if [ "$name" == "$statname" ]; then
                                okname=$name
                        fi
                done

                if [ "$okname" != "dummy" ]; then
                        echo $okname.label $name
                        echo $okname.warning $warn
                        echo $okname.critical $max
                fi
        done
        exit 0
fi;

readme="false"
cat /proc/user_beancounters | while read myid stuff; do
        line=""
        if [ "$myid" == "$ATTR:" ]; then
                readme="true"
                line="$stuff"
                echo $line
        else
                loid=`echo $myid | sed -e 's/.*:/:/'`
                if [ "$loid" == ":" ]; then
                        readme="false"
                fi
                if [ "$readme" == "true" ]; then
                        line="$myid $stuff"
                echo $line
                fi
        fi
done | while read name value x; do
        okname="dummy"
        for statname in $STATS; do
                if [ "$name" == "$statname" ]; then
                        okname=$name
                fi
        done
        if [ "$okname" != "dummy" ]; then
                echo $okname.value $value
        fi

done

This is not too performant but should do and the graphs are much more readable then the first solution.


Extended plugin from Jan Tomasek[edit]

  • Jan has posted another plugin on http://forum.openvz.org/index.php?t=msg&goto=15122, where I've fixed two things:
    1. "exit 0" in the "config" block
    2. Replaced "vals=($str); echo ${vals[0]}" with "echo ${str%% *}" (the former was causing problems I don't remember anymore)
  • v1.3.2 (2008/08/09)
    1. If only 1 variable is graphed, also display maxheld, barrier and limit
#!/bin/sh
#
# Munin's plugin to monitor OpenVZ bean counters.
#
# $Log$
# Revision 1.3 2007/07/19 12:57:00 Jan Tomasek <jan@tomasek.cz>
# * rewrited to work with /proc/bc/<VEID>/resources instead of
# /proc/user_beancounters, that simplified code and result
# is also bit faster.
# * added references to OpenVZ wiki
# Revision 1.3.1 2008/05/13 01:26:00 Daniel Hahler <http://daniel.hahler.de/>
# * Minor fixes
#   - "exit 0" in "config" block
#   - Use "echo ${str%% *}" in "suggest", instead of "vals=($str); echo ${vals[0]}"
#
# Revision 1.3.2 2008/08/09 12:30:00 Christian Rubbert <crubbert@xrc.de>
# * Feature
#   - If only 1 variable is graphed, also display maxheld, barrier and limit
#
# Original revision taken from:
# http://wiki.openvz.org/Monitoring_openvz_resources_using_munin
#
#%# family=auto
#%# capabilities=autoconf suggest
 
VEID=`basename $0 | sed -e 's/^vebc_.*_//'`;
STATS=`basename $0 | sed -e 's/^vebc_//' -e 's/_[0-9]*$//' -e 's/_/ /g'`
x=0; STATSCNT=`for i in $STATS; do x=$[$x+1]; done; echo $x`
 
if [ "$1" = "autoconf" ]; then
    if [ -r /proc/bc/0/resources ]; then
        echo yes
        exit 0
    else
        echo "no (/proc/bc/0/resources not found)"
        exit 1
    fi
fi
 
if [ "$1" = "suggest" ]; then
    if [ -r /proc/bc/0/resources ]; then
        cat /proc/bc/0/resources | while read str; do
        # Print everything before " "
        echo ${str%% *}
        done
        exit 0
    else
        exit 1
    fi
fi
 
if [ ! -f /proc/bc/$VEID/resources ]; then
    exit 0;
fi
 
if [ "$1" = "config" ]; then
    #echo "graph_order down up"
    echo "graph_title VE$VEID: $STATS"
    echo "graph_vlabel bean counters"
    echo "graph_category VE$VEID"

    # Note on URLs. General graph info is by munin version 1.2.5
    # accepted even with HTML code. But for value.info it escapes URL,
    # I expect that authors of munin will note that in future and put
    # escaping even for graph.info.
    echo "graph_info VE bean counters info. Documentation of the OpenVZ resource management is located at <a href=\"
http://wiki.openvz.org/UBC\">http://wiki.openvz.org/UBC</a>."
 
    cat /proc/bc/$VEID/resources | while read name value top warn max stuff ; do
        for statname in $STATS; do
        if [ "$name" = "$statname" ]; then
            URL="http://wiki.openvz.org/$name"
            if [ "$warn" = "0" ]; then
                warn=$max
            fi
            echo $name.label $name
            echo $name.warning $warn
            echo $name.critical $max
            echo $name.info Description of this resource is located at $URL
        fi
        done
    done

    if [ "$STATSCNT" = "1" ]; then
        echo maxheld.label Maxheld
        echo maxheld.draw LINE2
        echo maxheld.info Maximum value
        echo barrier.label Barrier
        echo barrier.draw LINE2
        echo barrier.info Barrier
        echo limit.label Limit
        echo limit.draw LINE2
        echo limit.info Limit
    fi
    exit 0
fi;
 
cat /proc/bc/$VEID/resources | while read name value top warn max stuff ; do
    for statname in $STATS; do
    if [ "$name" = "$statname" ]; then
        echo $name".value "$value;

	if [ "$STATSCNT" = "1" ]; then
	    echo maxheld.value $top
	    echo barrier.value $warn
	    echo limit.value $max 
	fi
    fi
    done
done

Munin plugin setup[edit]

Run as root[edit]

Please note, you have to configure plugin to run as root. Therefore, add the following to /etc/munin/plugin-conf.d/ somewhere:

 [vebc*]
 user root

Installing the plugins[edit]

There's a single plugin file, which can be installed several times and can put several values into the same graph. You should install the plugin from above to e.g. /usr/local/share/munin/plugins/vebc_ and then put symlinks to there from /etc/munin/plugins. The following script allows you to handle this easily: To install this, you can use the following script:

#!/bin/bash

FILE=`mktemp /tmp/ln-vebc-XXXXXX`

cd /etc/munin/plugins

for resources in kmemsize \
    lockedpages_privvmpages_shmpages_physpages_vmguarpages_oomguarpages \
    numproc \
    numtcpsock_numflock_numpty_numsiginfo_numothersock_numiptent \
    tcpsndbuf_tcprcvbuf_othersockbuf_dgramrcvbuf \
    dcachesize \
    numfile
do
    for VE in 0 `/usr/sbin/vzlist | sed "s/^ *//" |grep '^[0-9]' | cut -f 1 -d " "` ; do
        ln -sf /usr/local/share/munin/plugins/vebc_ "vebc_"$resources"_"$VE
        echo "vebc_"$resources"_"$VE >> $FILE
    done
done

# remove no longer deserved links (ie. links pointing to machines
# which were destroyed or stoped)

find -type l -name vebc_\* | sed "s/\.\///" | while read LN; do
    if grep ^$LN$ $FILE >/dev/null; then
        true
    else
        rm $LN
    fi
done

rm $FILE


Alternative: Using vzlist[edit]

This simple script will generate an overview of the requested stat from all VE's. When using vzlist your saved from any changes to the beancounters.

Usage[edit]

Simply append the variable you want to monitor:

ln -s /usr/share/munin/plugins/openvz_ /etc/munin/plugins/openvz_physpages
#!/bin/sh
#
# Munin's plugin to monitor OpenVZ bean counters.
#
# $Log$
# 2008/08/14 Rene Weselowski <http://www.dead.at>
#
#%# family=auto
#%# capabilities=autoconf

ATTRIBUTE=`basename $0 | sed 's/^openvz_//g'`


if [ "$1" = "autoconf" ]; then
        echo yes
        exit 0
fi

if [ "$1" = "config" ]; then
        echo "graph_title $ATTRIBUTE"
        echo "graph_args --base 1000 -l 0"
        echo "graph_scale yes"
        echo "graph_vlabel $ATTRIBUTE Value"
        echo "graph_category openvz"
        echo "graph_info This graph shows OpenVZ: $ATTRIBUTE"
        vzlist -a -H -o hostname | awk '{gsub(/\./,"_",$1)
        print("'$ATTRIBUTE'"$1".label "$1"\n" \
        "'$ATTRIBUTE'"$1".info '$ATTRIBUTE' for VE"$1)}'
        exit 0
fi

vzlist -a -H -o hostname,$ATTRIBUTE | awk '{gsub(/\./,"_",$1)
         print("'$ATTRIBUTE'"$1".value "$2)}'