Monday, April 30, 2012

Monitoring tool - install icinga and icinga-web

I've been using a monitoring tool, Nagios for several years. Though it is a classic monitoring tool and still in general use, icinga has several advantages compared with nagios, which shows here.

And the architecture of  Icinga is show in picture , here.

As Nagios is convertible with Icinga (it's not sure 100%), I am going to try migrate from Nagios-3.x to Icinga-1.x. The official says, here.

We try to be compatible with the configuration files of the current Nagios 3.x versions so there should be very little you have to do to “upgrade” from Nagios 3.x to Icinga 1.x. Assuming you’ve already installed Nagios from source code as described in the Nagios quickstart guide, you can install Icinga quite easily.

I am going to show how to install Icinga and migrate the current nagios-3.x configuration files in serial form.




  • Overview of System Structure
    • OS
    OS CentOS release 5.8 (Final)
    Kernel 2.6.18-274.el5 x86_64
    Language ja_JP.UTF-8
    • middleware
    role package
    WEB httpd-2.2.10
    DB mysql-5.5.21
    Scripting Language(for nagios-plugins) perl-5.14.2
    Scripting Language(for icinga-web) php-5.3.10
    icinga core icinga-1.6.1
    plugin nagios-plugins-1.4.15
    icinga frontend icinga-web-1.6.2
    icinga report icinga-reports-1.6.0
    reporting engine(for icinga-reports) jasperreports-server-cp-4.2.1
    graph(nagios addon) pnp4nagios-0.6.16
    • Directory structure of the middleware avobe
    /usr/local/
    |-- httpd -> /usr/local/httpd-2.2.22/
    |-- httpd-2.2.22/
    |-- icinga -> /usr/local/icinga-1.6.1/
    |-- icinga-1.6.1/
    |-- icinga-idoutils/
    |-- icinga-web -> /usr/local/icinga-web-1.6.2/
    |-- icinga-web-1.6.2/
    |-- jasperreports-server-cp-4.2.1/
    |-- mysql -> /usr/local/mysql-5.5.21/
    |-- mysql-5.5.21/
    |-- perl -> /usr/local/perl-5.14.2/
    |-- perl-5.14.2/
    |-- php -> /usr/local/php-5.3.10/
    `-- php-5.3.10/


 I am showing the process of installing icinga and icinga-web.

Icinga-core + NODUtils

  •  add user and group
# groupadd icinga && groupadd icinga-cmd
# useradd -G icinga-cmd,apache icinga 
  • install compiler, net-snmp and other dependet libaries
    $arch defines its architecture, i386 or x86_64. Here is x86_64(64bit OS)
# arch=$(uname -i)
# yum -y install gcc.${arch} glibc.${arch} glibc-common.${arch} gd.${arch} gd-devel
# yum -y install libjpeg.${arch} libjpeg-devel.${arch} libpng.${arch} libpng-devel.${arch} libdbi-devel.${arch}
# yum -y install net-snmp.${arch} net-snmp-devel.${arch} net-snmp-utils
  • install libdbi-drivers libdbi-dbd-mysql
# yum -y install libdbi-drivers.${arch} libdbi-dbd-mysql.${arch}
  • install icinga-core + idoutils
$ cd /usr/local/src/
$ curl "http://downloads.sourceforge.net/project/icinga/icinga/1.6.1/icinga-1.6.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Ficinga%2F&ts=1335254872&use_mirror=jaist" | tar zx
$ cd icinga-1.6.1
$ ./configure \
--prefix=/usr/local/icinga-1.6.1 \
--enable-nanosleep \
--enable-event-broker \
--enable-idoutils \
--enable-embedded-perl \
--enable-ssl \
--with-icinga-user=icinga \
--with-icinga-group=icinga \
--with-command-user=icinga \
--with-command-group=icinga-cmd \
--with-web-user=apache \
--with-web-group=apache \
--with-cgiurl=/icinga/cgi-bin \
--with-htmurl=/icinga \
--with-httpd-conf=/usr/local/httpd/conf/extra \
--with-checkresult-dir=/var/spool/checkresults \
--with-log-dir=/var/log/icinga \
--with-cgi-log-dir=/var/log/icinga \
--with-init-dir=/etc/init.d/ \
--with-lockfile=/var/run/icinga \
--with-ssl
$ make all
# make install-init
# make install-config
# make install-webconf
# make install-idoutils
# make install-commandmode
# make install
  • make a link with the prefix directory
# ln -s /usr/local/icinga /usr/local/icinga

  • setup configration, and enable the idomod event broker module
    The official shows the command below, which moves the sample configuration file, but ido2db.cfg and idmod.cfg are created.
# cd /usr/local/icinga/etc
# mv ido2db.cfg-sample ido2db.cfg
# mv idomod.cfg-sample idomod.cfg
# mv modules/idoutils.cfg-sample modules/idoutils.cfg
  • create mysql database
# mysqladmin -uroot create icinga
# mysql -uroot mysql -e "
GRANT USAGE ON *.* TO 'icinga'@'localhost'
   IDENTIFIED BY 'icinga'
   WITH MAX_QUERIES_PER_HOUR 0
   MAX_CONNECTIONS_PER_HOUR 0
   MAX_UPDATES_PER_HOUR 0;
GRANT ALL ON icinga.* TO 'icinga'@'localhost';
FLUSH PRIVILEGES;"
  • import database scheme
# mysql -uicinga -picinga icinga < /usr/local/src/icinga-1.6.1/module/idoutils/db/mysql/mysql.sql
  • install nagios-plugins
$ curl "http://jaist.dl.sourceforge.net/project/nagiosplug/nagiosplug/1.4.15/nagios-plugins-1.4.15.tar.gz" | tar xz
$ cd nagios-plugins-1.4.15
./configure \
--prefix=/usr/local/icinga \
--with-nagios-user=icinga \
--with-nagios-group=icinga \
--with-openssl \
--with-perl=/usr/local/perl/bin/perl
$ make
# make install

Icinga-web

  • install icinga-web
$ cd /usr/local/src/
$ curl "http://downloads.sourceforge.net/project/icinga/icinga-web/1.6.2/icinga-web-1.6.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Ficinga%2Ffiles%2Ficinga-web%2F1.6.2%2F&ts=1335497712&use_mirror=jaist" | tar zx
$ cd icinga-web-1.6.2
$ ./configure \
--prefix=/usr/local/icinga-web-1.6.2 \
--with-web-user=apache \
--with-web-group=apache \
--with-web-path=/icinga-web \
--with-web-apache-path=/usr/local/httpd/conf/extra \
--with-db-type=mysql \
--with-db-host=localhost \
--with-db-port=3306 \
--with-db-name=icinga_web \
--with-db-user=icinga_web \
--with-db-pass=icinga_web \
--with-db-socket=/tmp/mysql.sock \
--with-api-host=localhost \
--with-api-port=3306 \
--with-api-db-user=icinga \
--with-api-db-pass=icinga \
--with-api-db-name=icinga \
--with-api-db-prefix=icinga_ \
--with-api-cmd-file=/usr/local/icinga/var/rw/icinga.cmd \
--with-log-dir=/var/log/icinga \
--with-enable-sla
$ make
# make install
# make install-apache-config
# make install-javascript
  • test php modules' dependency
# make testdeps
/usr/local/bin/php bin/testdeps.php
Icinga-web dependencies (running 23 test)

PHP (core) tests
        1/23 Test php version >= 5.2.3 (REQUIRED) ... (version 5.3.10 >= 5.2.3) OK
        2/23 Test for PEAR (REQUIRED) ... OK

PHP extensions
        3/23 Test php5-xsl (REQUIRED) ... (xsl found v0.1) OK
        4/23 Test php5-ldap ... (ldap found) OK
        5/23 Test php5-pdo (REQUIRED) ... (PDO found v1.0.4dev) OK
        6/23 Test php5-dom (REQUIRED) ... (dom found v20031129) OK
        7/23 Test php5-session (REQUIRED) ... (session found) OK
        8/23 Test php5-spl (REQUIRED) ... (SPL found v0.2) OK
        9/23 Test php5-pcre (REQUIRED) ... (pcre found) OK
        10/23 Test php5-tokenizer (REQUIRED) ... (tokenizer found v0.1) OK
        11/23 Test php5-libxml (REQUIRED) ... (libxml found) OK
        12/23 Test php5-reflection (REQUIRED) ... (Reflection found v$Revision: 321634 $) OK
        13/23 Test php5-gettext (REQUIRED) ... (gettext found) OK

Optional pdo drivers
        14/23 Test php5-pdo-mysql ... (pdo_mysql found v1.0.2) OK
        15/23 Test php5-pdo-pgsql ... (Extension pdo_pgsql does not exist) FAIL

Optional php extension
        16/23 Test php5-soap ... (soap found) OK
        17/23 Test php5-xmlrpc ... (xmlrpc found v0.51) OK
        18/23 Test php5-iconv ... (iconv found) OK
        19/23 Test php5-gd ... (gd found) OK
        20/23 Test php5-ctype ... (ctype found) OK
        21/23 Test php5-json ... (json found v1.2.1) OK
        22/23 Test php5-hash ... (hash found v1.0) OK
        23/23 Test php.ini memory_limit ... (memory_limit='134217728') OK

All over result: PASS (required 12/12, optional 10/11, all 22/23, time 0.02s)

Exit (status=0)
  • create mysql database and user for icinga-web
# mysqladmin -uroot create icinga_web
# mysql -uroot mysql -e "
GRANT USAGE ON *.* TO 'icinga_web'@'localhost'
   IDENTIFIED BY 'icinga_web'
   WITH MAX_QUERIES_PER_HOUR 0
   MAX_CONNECTIONS_PER_HOUR 0
   MAX_UPDATES_PER_HOUR 0;
GRANT ALL ON icinga_web.* TO 'icinga_web'@'localhost';
FLUSH PRIVILEGES;"
  • initialize database
# make db-initialize
cd ./etc/ && ../bin/phing -f build.xml db-initialize
Buildfile: /usr/local/src/icinga-web-1.6.2/etc/build.xml
 [property] Loading /usr/local/src/icinga-web-1.6.2/etc/build.properties

icinga-web > db-initialize:

[phingcall] Calling Buildfile '/usr/local/src/icinga-web-1.6.2/etc/build.xml' with target 'db-create'
 [property] Loading /usr/local/src/icinga-web-1.6.2/etc/build.properties

icinga-web > prepare:

icinga-web > db-prepare:

Use the db-user specified in icinga-webs database.xml (Needs create Database/create Table rights)?(n,y)? y

icinga-web > db-create:

BUILD FINISHED

Total time: 3.4069 seconds
  • Test the syntax of icinga.cfg
    Though it is necessary to run Icinga daemon with the correct syntax of icinga.cfg and other related configuration files, such as nagios.cfg, commands.cfg, services.cfg, and so on, I am going to try show them later.
 # icinga -v /usr/local/icinga/etc/icinga.cfg 

Icinga 1.6.1
Copyright (c) 2009-2011 Icinga Development Team (http://www.icinga.org)
Copyright (c) 2009-2011 Nagios Core Development Team and Community Contributors
Copyright (c) 1999-2009 Ethan Galstad
Last Modified: 12-02-2011
License: GPL

Reading configuration data...
   Read main config file okay...
Processing object config directory '/usr/local/icinga/etc/modules'...
Processing object config file '/usr/local/icinga/etc/modules/idoutils.cfg'...
Processing object config directory '/usr/local/icinga/etc/objects'...
Processing object config file '/usr/local/icinga/etc/objects/commands.cfg'...
Processing object config file '/usr/local/icinga/etc/objects/contacts.cfg'...
Processing object config file '/usr/local/icinga/etc/objects/timeperiods.cfg'...
Processing object config file '/usr/local/icinga/etc/objects/nagios_perfparse.cfg'...
Processing object config file '/usr/local/icinga/etc/objects/templates.cfg'...
Processing object config directory '/usr/local/icinga/etc/templates'...
Processing object config file '/usr/local/icinga/etc/templates/hosts.cfg'...
Processing object config file '/usr/local/icinga/etc/templates/services_mon.cfg'...
   Read object config files okay...

Running pre-flight check on configuration data...

Checking services...
        Checked 32 services.
Checking hosts...
        Checked 1 hosts.
Checking host groups...
        Checked 1 host groups.
Checking service groups...
        Checked 0 service groups.
Checking contacts...
        Checked 1 contacts.
Checking contact groups...
        Checked 1 contact groups.
Checking service escalations...
        Checked 0 service escalations.
Checking service dependencies...
        Checked 0 service dependencies.
Checking host escalations...
        Checked 0 host escalations.
Checking host dependencies...
        Checked 0 host dependencies.
Checking commands...
        Checked 47 commands.
Checking time periods...
        Checked 3 time periods.
Checking modules...
        Checked 1 modules.
Checking for circular paths between hosts...
Checking for circular host and service dependencies...
Checking global event handlers...
Checking obsessive compulsive processor commands...
Checking misc settings...

Total Warnings: 0
Total Errors:   0

Things look okay - No serious problems were detected during the pre-flight check
  • start icinga daemon
# /etc/init.d/icinga start
Running configuration check...OK
Starting icinga: Starting icinga done.
  • restart apache daemon
# /etc/init.d/httpd restart
  • Top page after logging in

  • A table of items’ status
Next will be how to install icinga-reports, pnp4nagios or about the details of configuration and init script.

Sunday, April 1, 2012

NFSv3/v4 monitoring

I introduced how to setup NFSv3/v4 Server in the last session, NFSv3/v4 setup & monitoring. Then, I'd like to show you how to monitor NFSv3/v4 servers with nagios plugins.

NFSv3 NFS Server

  • verify the status with nagios plugins, check_rpc while nfs and portmap are running
# /etc/init.d/nfs status
rpc.mountd (pid 21781) を実行中...
nfsd (pid 21778 21777 21776 21775 21774 21773 21772 21771) を実行中...

# /etc/init.d/portmap status
portmap (pid 10609) を実行中...

# ps awuxc | egrep '(nfs|portmap|idmapd)'
rpc      29800  0.0  0.0   8072   676 ?        Ss   20:42   0:00 portmap
root     29915  0.0  0.0      0     0 ?        S<   20:42   0:00 nfsd4
root     29917  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29918  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29919  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29920  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29921  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29922  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29923  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd
root     29924  0.0  0.0      0     0 ?        S    20:42   0:00 nfsd

# rpcinfo -p
  program vers proto   port
   100000    2   tcp    111  portmapper
   100000    2   udp    111  portmapper
   100024    1   udp    773  status
   100024    1   tcp    776  status
   100003    2   udp   2049  nfs
   100003    3   udp   2049  nfs
   100003    4   udp   2049  nfs
   100021    1   udp  53048  nlockmgr
   100021    3   udp  53048  nlockmgr
   100021    4   udp  53048  nlockmgr
   100003    2   tcp   2049  nfs
   100003    3   tcp   2049  nfs
   100003    4   tcp   2049  nfs
   100021    1   tcp  37837  nlockmgr
   100021    3   tcp  37837  nlockmgr
   100021    4   tcp  37837  nlockmgr
   100005    3   udp    892  mountd
   100005    3   tcp    892  mountd
# check_rpc -H localhost -t -C nfs
OK: RPC program nfs version 2 version 3 version 4 tcp running
  • verify the status while nfs is stopping
# /etc/init.d/nfs stop
NFS mountd を終了中:                                       [  OK  ]
NFS デーモンを終了中:                                      [  OK  ]
NFS サービスを終了中:                                      [  OK  ]
# check_rpc -H localhost -t -C nfs
CRIICAL: RPC program nfs  tcp is not running
  • verify the status while nfs and portmap are stopping
# /etc/init.d/portmap stop
portmap を停止中:                                          [  OK  ]
# check_rpc -H localhost -t -C nfs
CRITICAL: RPC program nfs  tcp is not running

NFSv4 NFS Server

We are going to use the nagios plugins, check_nfs4.0.2.pl to verify the status of nfs v4 server.

  • verify the status with nagios plugins, check_nfsv4.0.2.pl while nfs, portmap, and rpcidmapd are running
# /etc/init.d/nfs status
rpc.mountd (pid 21781) を実行中...
nfsd (pid 21778 21777 21776 21775 21774 21773 21772 21771) を実行中...

# /etc/init.d/portmap status
portmap (pid 10609) を実行中...

# /etc/init.d/rpcidmapd status
rpc.idmapd (pid 18346) を実行中...

# ps awuxc | egrep '(nfs|portmap|idmapd)'
root      8102  0.0  0.0      0     0 ?        S<   Jan02   0:00 nfsiod
rpc      10609  0.0  0.0   8052   580 ?        Ss   11:44   0:00 portmap
root     18346  0.0  0.0  55180  1008 ?        Ss   12:20   0:00 rpc.idmapd
root     21770  0.0  0.0      0     0 ?        S<   12:35   0:00 nfsd4
root     21771  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21772  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21773  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21774  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21775  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21776  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21777  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd
root     21778  0.0  0.0      0     0 ?        S    12:35   0:00 nfsd

# rpcinfo -p
   program vers proto   port
    100000    2   tcp    111  portmapper
    100000    2   udp    111  portmapper
    100011    1   udp    870  rquotad
    100011    2   udp    870  rquotad
    100011    1   tcp    873  rquotad
    100011    2   tcp    873  rquotad
    100003    4   tcp   2049  nfs
    100021    1   udp  32769  nlockmgr
    100021    3   udp  32769  nlockmgr
    100021    4   udp  32769  nlockmgr
    100021    1   tcp  32803  nlockmgr
    100021    3   tcp  32803  nlockmgr
    100021    4   tcp  32803  nlockmgr
    100005    3   udp    757  mountd
    100005    3   tcp    760  mountd
# ./check_nfs4.0.2.pl -v
OK: nfsd cpu = 0% ; nfsd threads = 8 ; nfsd used threads <= 10% ; Server badcalls = 19 ; Server badauth = 19 |nfsd_cpu=0% nfsd_used_threads=10% io_read=0% io_write=0%
  • verify the status while rpcidmapd is stopping
# /etc/init.d/rpcidmapd stop
Stopping RPC idmapd:                                       [  OK  ]
# ./check_nfs4.0.2.pl -v
CRITICAL: nfsd cpu = 0% ; nfsd threads = 8 ; nfsd used threads <= 10% ; daemon idmapd is not running ; Server badcalls = 19 ; Server badauth = 19 |nfsd_cpu=0% nfsd_used_threads=10% io_read=0% io_write=0%
  • verify the status while rpcidmapd and nfs are stopping
# /etc/init.d/nfs stop
NFS mountd を終了中:                                       [  OK  ]
NFS デーモンを終了中:                                      [  OK  ]
NFS サービスを終了中:                                      [  OK  ]
# ./check_nfs4.0.2.pl -v
CRITICAL: nfsd cpu = 0% ; nfsd threads = 0 ; nfsd used threads <= 10% ; daemons idmapd nfsd mountd are not running ; Server badcalls = 19 ; Server badauth = 19 |nfsd_cpu=0% nfsd_used_threads=10% io_read=0% io_write=0%
  • verify the status while rpcidmapd, nfs and portmap are stopping
# /etc/init.d/portmap stop
portmap を停止中:                                          [  OK  ]
# ./check_nfs4.0.2.pl -v
OK: nfsd cpu = 0% ; nfsd threads = 8 ; nfsd used threads <= 10% ; Server badcalls = 19 ; Server badauth = 19 |nfsd_cpu=0% nfsd_used_threads=10% io_read=0% io_write=0%

NFSv4 NFS Server

check_nfsv4.0.2.pl is enable to verify the status of nfsv4 client, too.

  • verify the status while portmap and rpcidmapd are running
# /etc/init.d/portmap status
portmap (pid 30107) を実行中...

# /etc/init.d/rpcidmapd status
rpc.idmapd (pid 30002) を実行中...
# ./check_nfs4.0.2.pl -i -v
OK: |
  • verify the status while rpcidmapd is stopping
# /etc/init.d/rpcidmapd stop
Stopping RPC idmapd:                                       [  OK  ]
# ./check_nfs4.0.2.pl -i -v
CRITICAL: daemon idmapd is not running |
  • verify the status while portmap is stopping
# /etc/init.d/portmap stop
portmap を停止中:                                          [  OK  ]
# ./check_nfs4.0.2.pl -i -v
OK: |
  • verify the status while portmap and rpcidmapd are stopping
# /etc/init.d/rpcidmapd stop
Stopping RPC idmapd:                                       [  OK  ]

# /etc/init.d/portmap stop
portmap を停止中:                                          [  OK  ]
# ./check_nfs4.0.2.pl -i -v
CRITICAL: daemon idmapd is not running |


These are the examples of services.cfg in the nagios configuration file, below.
  • common setting with visudo for nagios user between nfs server and client.
nagios          ALL=(ALL)       NOPASSWD: ALL
  • commands.cfg for check_by_ssh
define command{
        command_name    check_by_ssh_pub
        command_line    $USER1$/check_by_ssh -H $HOSTADDRESS$ -i /usr/local/nagios/.ssh/id_rsa -l nagios -p $ARG1$ -t $ARG2$ -C $ARG3$
}

  • NFSv3 Server
define service{
        use                     generic-service
        host_name               nfsv3_server
        service_description     NFSv3:Server
        check_command           check_by_ssh_pub!22!60!"/usr/local/nagios/libexec/check_rpc -H localhost -t -C nfs"
}
  • NFSv4 Server
define service{
        use                     generic-service
        host_name               nfsv4_server
        service_description     NFSv4:Server
        check_command           check_by_ssh_pub!22!60!"/usr/bin/sudo /usr/local/nagios/libexec/check_nfs4.0.2.pl -v"
}
  • NFSv4 Client
define service{
        use                     generic-service
        host_name               nfsv4_client
        service_description     NFSv4:Client
        check_command           check_by_ssh_pub!22!60!"/usr/bin/sudo /usr/local/nagios/libexec/check_nfs4.0.2.pl -v"
}