Linux

ApacheからFluentdにログを送信

| by 優

 

WEBサーバ

TCP, UDPの24224番ポートを解放しておくこと。

 

# vi /etc/sysconfig/selinux


SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted

 

 

# yum install httpd httpd-devel php php-mysql mysql mariadb-server mariadb

# systemctl start httpd
# systemctl enable httpd

 

# yum install unzip wget

# yum groupinstall "Development Tools" "Base"

# yum install gcc gcc-c++ pcre-devel zlib-devel make wget openssl-devel libxml2 libxml2-devel libxslt-devel libxslt libxslt-devel gd-devel perl-ExtUtils-Embed GeoIP-devel gperftools-devel flex

# yum install gcc tcl readline-devel

 

# vi /etc/security/limits.conf


root soft nofile 65536
root hard nofile 65536
* soft nofile 65536
* hard nofile 65536

 

# vi /etc/sysctl.conf

net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.ip_local_port_range = 10240    65535

 

# reboot now

 

td-agentの中身

  • Ruby
  • コアライブラリ
  • fluentd とプラグイン

 

# curl -L https://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh


Installed:
  td-agent.x86_64 0:2.3.6-0.el7

Dependency Installed:
  at.x86_64 0:3.1.13-22.el7                                   avahi-libs.x86_64 0:0.6.31-17.el7
  bc.x86_64 0:1.06.95-13.el7                                  cups-client.x86_64 1:1.6.3-29.el7
  cups-libs.x86_64 1:1.6.3-29.el7                             ed.x86_64 0:1.9-4.el7
  m4.x86_64 0:1.4.16-10.el7                                   mailx.x86_64 0:12.5-16.el7
  patch.x86_64 0:2.7.1-8.el7                                  psmisc.x86_64 0:22.20-15.el7
  redhat-lsb-core.x86_64 0:4.1-27.el7.centos.1                redhat-lsb-submod-security.x86_64 0:4.1-27.el7.centos.1
  spax.x86_64 0:1.5.2-13.el7                                  time.x86_64 0:1.7-45.el7

Complete!

Installation completed. Happy Logging!

NOTE: In case you need any of these:
  1) security tested binary with a clear life cycle management
  2) advanced monitoring and management
  3) support SLA
Please check Fluentd Enterprise (https://www.treasuredata.com/fluentd/).

 

# systemctl start td-agent
# systemctl enable td-agent

 

# sudo td-agent-gem install fluent-plugin-elasticsearch

 

[root@ip-172-31-16-106 ~]# curl -X POST -d 'json={"json":"message"}' http://localhost:8888/debug.test

[root@ip-172-31-16-106 ~]# tail -f /var/log/td-agent/td-agent.log


  </source>
  <source>
    @type debug_agent
    bind 127.0.0.1
    port 24230
  </source>
</ROOT>
2018-01-04 10:19:25 +0000 [info]: listening fluent socket on 0.0.0.0:24224
2018-01-04 10:19:25 +0000 [info]: listening dRuby uri="druby://127.0.0.1:24230" object="Engine"
2018-01-04 10:27:51 +0000 debug.test: {"json":"message"}

 

# vi /etc/td-agent/td-agnet.conf


# 末尾に追記
<source>
  type tail
  path /var/log/httpd/access_log
  tag apache.access
  pos_file /var/log/td-agent/access_log.pos
  format apache2
</source>

<match apache.access>
  type file
  path /var/log/td-agent/httpd/access.log
  time_slice_format %Y%m%d
  time_slice_wait 10m
  compress gzip
</match>

 

# systemctl restart td-agent

 

 

# tail -f /var/log/td-agent/td-agent.log

2018-01-04 10:39:41 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:41 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:42 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:42 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:43 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:43 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:44 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:44 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:45 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:45 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:46 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:46 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:47 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:47 +0000 [error]: suppressed same stacktrace
2018-01-04 10:39:48 +0000 [error]: Permission denied @ rb_file_s_stat - /var/log/httpd/access_log
  2018-01-04 10:39:48 +0000 [error]: suppressed same stacktrace

エラー

 

# chmod o+x /var/log/httpd
# systemctl restart td-agent

 

# ls -laht /var/log/td-agent/httpd/

total 68K
-rw-r--r-- 1 td-agent td-agent 66K Jan  4 10:41 access.log.20180104.b561f0f5384a87574
drwxr-xr-x 2 td-agent td-agent  51 Jan  4 10:40 .
drwxr-xr-x 4 td-agent td-agent  75 Jan  4 10:38 ..

 

 

ログをログ集約サーバ Fluentd　172.31.23.254に送るようにする

# vi /etc/td-agent/td-agent.conf



# 末尾に追記
<source>
  type tail
  path /var/log/httpd/access_log
  tag apache.access
  pos_file /var/log/td-agent/access_log.pos
  format apache2
</source>

#<match apache.access>
#  type file
#  path /var/log/td-agent/httpd/access.log
#  time_slice_format %Y%m%d
#  time_slice_wait 10m
#  compress gzip
#</match>

<match **>
  type forward
  <server>
    host 172.31.23.254
    port 24224
  </server>
</match>

 

 

 

 

ログ集約サーバ Fluentd

 

# curl -L http://toolbelt.treasuredata.com/sh/install-redhat-td-agent2.sh | sh

# sudo td-agent-gem install fluent-plugin-elasticsearch

 

# vi /etc/sysconfig/selinux


SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted

 

# yum install httpd httpd-devel php php-mysql mysql mariadb-server mariadb

# systemctl start httpd
# systemctl enable httpd

 

# yum install unzip wget

# yum groupinstall "Development Tools" "Base"

# yum install gcc gcc-c++ pcre-devel zlib-devel make wget openssl-devel libxml2 libxml2-devel libxslt-devel libxslt libxslt-devel gd-devel perl-ExtUtils-Embed GeoIP-devel gperftools-devel flex

# yum install gcc tcl readline-devel

 

# cp /etc/td-agent/td-agent.conf /etc/td-agent/td-agent.conf.org

# rm -f /etc/td-agent/td-agent.conf

 

# vi /etc/td-agent/td-agent.conf


<source>
    type forward
    port 24224
    bind 0.0.0.0
</source>

<match **>
    type file
    path /var/log/fluentd/out
</match>

 

# mkdir /var/log/fluentd
# chown td-agent:td-agent /var/log/fluentd

 

# systemctl restart td-agent

# systemctl enable td-agent
td-agent.service is not a native service, redirecting to /sbin/chkconfig.
Executing /sbin/chkconfig td-agent on

 

 

関連記事 - More from my site -

Amazonおすすめ

iPad 9世代 2021年最新作

iPad 9世代出たから買い替え。安いぞ！🐱 初めてならiPad。Kindleを外で見るならiPad mini。ほとんどの人には通常のiPadをおすすめします＞＜

コメントを残す

メールアドレスが公開されることはありません。 * が付いている欄は必須項目です

日本語が含まれない投稿は無視されますのでご注意ください。（スパム対策）