filebeat镜像制作

vim docker-entrypoint.sh

#!/bin/bash

ENV=${ENV:-"test"}
PROJ_NAME=${PROJ_NAME:-"no-define"}
MULTILINE=${MULTILINE:-"^\d{2}"}

cat > /etc/filebeat.yaml << EOF
filebeat.inputs:
- type: log
  fields_under_root: true
  fields:
    topic: logm-${PROJ_NAME}
  paths:
    - /logm/*.log
    - /logm/*/*.log
    - /logm/*/*/*.log
    - /logm/*/*/*/*.log
    - /logm/*/*/*/*/*.log
  scan_frequency: 120s
  max_bytes: 10485760
  multiline.pattern: '$MULTILINE'
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 100
- type: log
  fields_under_root: true
  fields:
    topic: logu-${PROJ_NAME}
  paths:
    - /logu/*.log
    - /logu/*/*.log
    - /logu/*/*/*.log
    - /logu/*/*/*/*.log
    - /logu/*/*/*/*/*.log
    - /logu/*/*/*/*/*/*.log
output.kafka:
  hosts: ["192.168.51.45:9092","192.168.51.46:9092","192.168.51.47:9092"]
  topic: k8s-$ENV-%{[topic]}
  version: 2.0.0
  required_acks: 0
  max_message_bytes: 10485760
EOF

set -xe

# If user don't provide any command
# Run filebeat
if [[ "$1" == "" ]]; then
     exec filebeat  -c /etc/filebeat.yaml 
else
    # Else allow the user to run arbitrarily commands like bash
    exec "$@"
fi

vim Dockerfile

FROM centos:7

ENV FILEBEAT_VERSION=7.4.1

RUN set -x && \
  curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo && \
  yum install -y wget && \
  wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-${FILEBEAT_VERSION}-linux-x86_64.tar.gz -O /opt/filebeat.tar.gz && \
  cd /opt && \
  tar xzvf filebeat.tar.gz && \
  cd filebeat-* && \
  cp filebeat /bin/ && \
  cd /opt && \
  rm -rf filebeat* && \
  yum clean all

COPY docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]

logstash配置

vim /etc/logstash/conf.d/k8s-pro.conf

input {
  kafka {
    bootstrap_servers => "192.168.51.45:9092,192.168.51.46:9092,192.168.51.47:9092"
    consumer_threads => 4
    group_id => "k8s_pro"
    topics_pattern => "k8s-pro-.*"
  }
}

filter {
  json {
    source => "message"
  }
}

output {
  elasticsearch {
    hosts => ["192.168.51.45:9200","192.168.51.46:9200","192.168.51.47:9200"]
    index => "k8s-pro-%{+YYYY.MM.DD}"
  }
}

pod收集日志配置

vim deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-xia
  namespace: xia
spec:
  replicas: 1
  selector:
    matchLabels:
      appname: nginx-xia
  template:
    metadata:
      labels:
        appname: nginx-xia
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: appname
                  operator: In
                  values:
                  - nginx-xia
              topologyKey: kubernetes.io/hostname
            weight: 50
      containers:
      - name: nginx-xia
        image: harbor.com.cn/library/nginx:alpine-base
        ports:
        - containerPort: 80
        volumeMounts:
        - mountPath: /var/log/nginx
          name: logu
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /
            port: 80
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 200m
            memory: 512Mi
          limits:
            cpu: 200m
            memory: 512Mi
      - name: filebeat
        image: harbor.com.cn/library/filebeat:v7.4.1
        imagePullPolicy: IfNotPresent
        env:
        - name: ENV
          value: pro
        - name: PROJ_NAME
          value: nginx-xia
        volumeMounts:
        - mountPath: /logu
          name: logu
      volumes:
      - emptyDir: {}
        name: logu
---
apiVersion: v1   
kind: Service
metadata:
  annotations:
    blackbox_path: "/"
    blackbox_port: "80"
    blackbox_scheme: "http"
  name: nginx-xia
  namespace: xia
spec:
  selector:
    appname: nginx-xia
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
---
apiVersion: networking.k8s.io/v1 
kind: Ingress
metadata:
  name: nginx-xia
  namespace: xia
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/proxy-body-size: "1000m"
spec:
  rules:
    - host: xia.wuxingge.com
      http:
        paths:
          - path: /
            pathType: ImplementationSpecific
            backend:
              service:
                name: nginx-xia
                port:
                  number: 80

附

https://mp.weixin.qq.com/s/SBZRrF8QB7-O-mUfh40GAg