nginx-Ingress-k8s内部7层负载均衡器

root@slave001:~/ingress# cat default-backend.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: default-http-backend
  labels:
    k8s-app: default-http-backend
  namespace: kube-system
spec:
  replicas: 1
  selector:
   matchLabels:
     k8s-app: default-http-backend
  template:
    metadata:
      labels:
        k8s-app: default-http-backend
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: default-http-backend
        # Any image is permissable as long as:
        # 1. It serves a 404 page at /
        # 2. It serves 200 on a /healthz endpoint
        image: registry.cn-hangzhou.aliyuncs.com/hachikou/defaultbackend:1.0
        livenessProbe:
          httpGet:
            path: /healthz   #这个URI是 nginx-ingress-controller中nginx里配置好的localtion 
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 30   #30s检测一次/healthz
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
#        resources:
#          limits:
#            cpu: 10m
#            memory: 20Mi
#          requests:
#            cpu: 10m
#            memory: 20Mi
      nodeName: 192.168.192.152
---
apiVersion: v1
kind: Service     #为default backend 创建一个service
metadata:
  name: default-http-backend
  namespace: kube-system
  labels:
    k8s-app: default-http-backend
spec:
  ports:
  - port: 80
    targetPort: 8080
  selector:
    k8s-app: default-http-backend

root@slave001:~/ingress# cat nginx-ingress-controller-rbac.yml 
---
apiVersion: v1
kind: ServiceAccount    
metadata:
  name: nginx-ingress-serviceaccount #创建一个serveerAcount
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole   #这个ServiceAcount所绑定的集群角色
rules:
  - apiGroups:
      - "" 
    resources:    #此集群角色的权限，它能操作的API资源 
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
        - events
    verbs:
        - create
        - patch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:         
  name: nginx-ingress-role  #这是一个角色，而非集群角色 
  namespace: kube-system
rules:  #角色的权限 
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get
      - create
      - update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding       #角色绑定
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount #绑定在这个用户 
    namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding      #集群绑定
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount   #集群绑定到这个serviceacount
    namespace: kube-system   #集群角色是可以跨namespace，但是这里只指明给这个namespce来使用

root@slave001:~/ingress# cat nginx-ingress-controller.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-ingress-controller
  labels:
    k8s-app: nginx-ingress-controller
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
       k8s-app: nginx-ingress-controller
  template:
    metadata:
      labels:
        k8s-app: nginx-ingress-controller
    spec:
      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
      # like with kubeadm
      # hostNetwork: true #注释表示不使用宿主机的80口，
      terminationGracePeriodSeconds: 60
      hostNetwork: true  #表示容器使用和宿主机一样的网络
      serviceAccountName: nginx-ingress-serviceaccount #引用前面创建的serviceacount
      containers:   
      - image: registry.cn-hangzhou.aliyuncs.com/peter1009/nginx-ingress-controller:0.20.0      #容器使用的镜像
        name: nginx-ingress-controller  #容器名
        #readinessProbe:   #启动这个服务时要验证/healthz 端口10254会在运行的node上监听。 
        #  httpGet:
        #    path: /healthz
        #    port: 10254
        #    scheme: HTTP
        #livenessProbe:
        #  httpGet:
        #    path: /healthz
        #    port: 10254
        #    scheme: HTTP
        #  initialDelaySeconds: 60  #每隔10做健康检查 
        #  timeoutSeconds: 1
        ports:
        - containerPort: 80  
          hostPort: 80    #80映射到80
#        - containerPort: 443
#          hostPort: 443
        env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
          - name: POD_NAMESPACE
            valueFrom:
              fieldRef:
                fieldPath: metadata.namespace
        args:
        - /nginx-ingress-controller
        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
#        - --default-ssl-certificate=$(POD_NAMESPACE)/ingress-secret    #这是启用Https时用的
#      nodeSelector:  #指明运行在哪，此IP要和default backend是同一个IP
#        kubernetes.io/hostname: 10.3.1.17   #上面映射到了hostport80，确保此IP80，443没有占用.
# 
      nodeName: 192.168.192.152

root@slave001:~/ingress# cat demo-tomcat.yaml 
apiVersion: v1
kind: Service
metadata:
  name: tomcat
  namespace: default
spec:
  selector:
    app: tomcat
    release: canary
  ports:
  - name: http
    targetPort: 8080
    port: 8080
  - name: ajp
    targetPort: 8009
    port: 8009
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: tomcat
      release: canary
  template:
    metadata:
      labels:
        app: tomcat
        release: canary
    spec:
      containers:
      - name: tomcat
        image: 192.168.192.155:80/chuan/tomcat:8.5.34-jre8-alpine #tomcat:8.5.34-jre8-alpine
        ports:
        - name: http
          containerPort: 8080
          name: ajp
          containerPort: 8009

root@slave001:~/ingress# cat ingress-myapp.yaml 
apiVersion: extensions/v1beta1          #api版本
kind: Ingress           #清单类型
metadata:                #元数据
  name: ingress-myapp    #ingress的名称
  namespace: default     #所属名称空间
  annotations:           #注解信息
    kubernetes.io/ingress.class: "nginx" # 将配置传到ingress controller中的nginx配置中
spec:      #规格
  rules:   #定义后端转发的规则
  - host: www.chuan.com    #通过域名进行转发
    http:
      paths:       
      - path:       #配置访问路径，如果通过url进行转发，需要修改；空默认为访问的路径为"/"
        backend:    #配置后端服务
          serviceName: tomcat
          servicePort: 8080

root@slave001:~/ingress# kubectl describe ingress ingress-myapp 
Name:             ingress-myapp
Namespace:        default
Address:          
Default backend:  default-http-backend:80 (10.200.122.229:8080)
Rules:
  Host           Path  Backends
  ----           ----  --------
  www.chuan.com  
                    tomcat:8080 (10.200.111.38:8080,10.200.163.106:8080)
Annotations:     kubernetes.io/ingress.class: nginx
Events:          <none>

root@slave001:~/ingress# kubectl exec -it nginx-ingress-controller-559847d7f9-fpkq4 -n kube-system -- cat nginx.conf
....
    ## start server www.chuan.com
    server {
        server_name www.chuan.com ;
        
        listen 80;
        
        listen [::]:80;
        
        set $proxy_upstream_name "-";
        
        location / {
            
            set $namespace      "default";
            set $ingress_name   "ingress-myapp";
            set $service_name   "tomcat";
            set $service_port   "8080";
            set $location_path  "/";
            
            rewrite_by_lua_block {
                
                balancer.rewrite()
                
            }
            
            log_by_lua_block {
                
                balancer.log()
                
                monitor.call()
            }
            
            port_in_redirect off;
            
            set $proxy_upstream_name "default-tomcat-8080";
            
            client_max_body_size                    1m;
            
            proxy_set_header Host                   $best_http_host;
            
            # Pass the extracted client certificate to the backend
            
            # Allow websocket connections
            proxy_set_header                        Upgrade           $http_upgrade;
            
            proxy_set_header                        Connection        $connection_upgrade;
            
            proxy_set_header X-Request-ID           $req_id;
            proxy_set_header X-Real-IP              $the_real_ip;
            
            proxy_set_header X-Forwarded-For        $the_real_ip;
            
            proxy_set_header X-Forwarded-Host       $best_http_host;
            proxy_set_header X-Forwarded-Port       $pass_port;
            proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
            
            proxy_set_header X-Original-URI         $request_uri;
            
            proxy_set_header X-Scheme               $pass_access_scheme;
            
            # Pass the original X-Forwarded-For
            proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;
            
            # mitigate HTTPoxy Vulnerability
            # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
            proxy_set_header Proxy                  "";
            
            # Custom headers to proxied server
            
            proxy_connect_timeout                   5s;
            proxy_send_timeout                      60s;
            proxy_read_timeout                      60s;
            
            proxy_buffering                         off;
            proxy_buffer_size                       4k;
            proxy_buffers                           4 4k;
            proxy_request_buffering                 on;
            
            proxy_http_version                      1.1;
            
            proxy_cookie_domain                     off;
            proxy_cookie_path                       off;
            
            # In case of errors try the next upstream server before returning an error
            proxy_next_upstream                     error timeout;
            proxy_next_upstream_tries               3;
            
            proxy_pass http://upstream_balancer;
            
            proxy_redirect                          off;
            
        }
        
    }
    ## end server www.chuan.com
....

root@slave001:~/ingress# cat demo-tomcat2.yaml 
apiVersion: v1
kind: Service
metadata:
  name: tomcat2
  namespace: default
spec:
  selector:
    app: tomcat2
    release: canary2
  ports:
  - name: http
    targetPort: 8080
    port: 8080
  - name: ajp
    targetPort: 8009
    port: 8009
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy2
  namespace: default
spec:
  replicas: 2
  selector:
    matchLabels:
      app: tomcat2
      release: canary2
  template:
    metadata:
      labels:
        app: tomcat2
        release: canary2
    spec:
      containers:
      - name: tomcat2
        image: 192.168.192.155:80/chuan/tomcat:8.5.32-jre8-alpine
        ports:
        - name: http
          containerPort: 8080
          name: ajp
          containerPort: 8009

root@slave001:~/ingress# cat  ingress_multi-host.yaml 
apiVersion: extensions/v1beta1          #api版本
kind: Ingress
metadata:
  name: nginx-web
  namespace: default
  annotations:
    kubernetes.io/ingress.class: "nginx" ##将配置传到ingress controller中的nginx配置中
spec:
  rules:
  - host: www.chuan.com
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat
          servicePort: 8080

  - host: mobile.chuan.com
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat2
          servicePort: 8080

openssl genrsa -out tls.key 2048
openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=www.chuan.com
kubectl create secret tls tomcat-ingress-secret --cert=tls.crt --key=tls.key
kubectl get secret
kubectl describe secret tomcat-ingress-secret

root@slave001:~/ingress/tls# cat ingress-tomcat-tls.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-tomcat-tls
  namespace: default
  annotations:
    kubernetes.io/ingress.class: "nginx"  # 将配置传到ingress controller中的nginx配置中
spec:
  tls:
  - hosts:
    - www.chuan.com
    secretName: tomcat-ingress-secret
  rules:
  - host: www.chuan.com
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat
          servicePort: 8080

openssl genrsa -out tls.key 2048
openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=mobile.chuan.com
kubectl create secret tls tomcat-ingress-secret2 --cert=tls.crt --key=tls.key
kubectl get secret
kubectl describe secret tomcat-ingress-secret

root@slave001:~/ingress/tls/tls2# cat ingress-tomcat-tls.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-tomcat-tls
  namespace: default
  annotations:
    kubernetes.io/ingress.class: "nginx"  # 将配置传到ingress controller中的nginx配置中
spec:
  tls:
  - hosts:
    - www.chuan.com
    secretName: tomcat-ingress-secret
  rules:
  - host: www.chuan.com
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat
          servicePort: 8080
  tls:
  - hosts:
    - mobile.chuan.com
    secretName: tomcat-ingress-secret2
  rules:
  - host: mobile.chuan.com
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat2
          servicePort: 8080

root@slave001:~/ingress# cat ingress-url.yaml 
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
  name: nginx-web
  namespace: chuan
  annotations:
    kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型
    nginx.ingress.kubernetes.io/use-regex: "true" ##指定后面rules定义的path可以使用正则表达式
    nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" ##连接超时时间,默认为5s
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600" ##后端服务器回转数据超时时间,默认为60s
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600" ##后端服务器响应超时时间,默认为60s
    nginx.ingress.kubernetes.io/proxy-body-size: "10m" ##客户端上传文件，最大大小，默认为20m
    #nginx.ingress.kubernetes.io/rewrite-target: / ##URL重写
    nginx.ingress.kubernetes.io/app-root: /index.html
spec:
  rules:
  - host: www.chuan.com
    http:
      paths:
      - path: /url1
        backend:
          serviceName: tomcat1-service
          servicePort: 8080
      - path: /url2
        backend:
          serviceName: tomcat2-service 
          servicePort: 8080


  - host: mobile.chuan.com
    http:
      paths:
      - path: /url1
        backend:
          serviceName: tomcat1-service
          servicePort: 8080
      - path: /url2
        backend:
          serviceName: tomcat2-service
          servicePort: 8080