elasticsearch - 使用 NSSM 将多个配置文件作为 logstash 服务运行

Question

我正在使用 logstash-7.4 并使用 NSSM 将其作为服务运行。我有一个配置文件将数据摄取到 ElasticSearch 的索引（index_one），另一个配置文件将数据摄取到 ElasticSearch 的另一个索引（indiex_two）中。（注意：- 两个配置文件都是按不同的时间间隔和时间安排的）。我可以将这两个文件设置为具有两个不同名称的服务吗，例如 service_one 用于将 conf 文件摄取数据到 index_one 和 service_two 用于 conf 文件将数据摄取到 indiex_two。这样做是好事还是有更好的方法来做同样的事情。

以下是两个配置文件： config file1: #file:db.conf

input { 
    jdbc { 
        jdbc_driver_library => ""
        jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver" 
        jdbc_connection_string => "jdbc:oracle:thin:@nl0123vca0020.xyz-nl0123.abc.com:1521/pintu1"
        jdbc_user =>"usersys2"
        jdbc_password => "password"
        statement => "select name, id, address , col_1, col_2, col_3 from  demo_table_1"
        schedule => "0 */2 * * *"
        last_run_metadata_path => "E:/logstash-7.4.2/config/intouch_db_index_increment.txt"
        use_column_value => true
        tracking_column => "version"
    } 
}
filter {
    mutate {
        convert => {
            "contentid" => "string"
        }
    }
}
output{
    elasticsearch {
        hosts => ["http://***.***.119.199:9200"]
        index => "index_two"
        document_id =>"%{contentid}"
        user => "elastic" 
        password => "passwordes" 
    }
}

---

配置文件-2：-

input {
    jdbc {
        jdbc_driver_library => ""
        jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
        jdbc_connection_string => "jdbc:oracle:thin:@nl0123vca0020.xyz-nl0123.abc.com:1521/pintu1"
        jdbc_user =>"usersys2"
        jdbc_password => "password"
        statement  => "select autosuggestid,userid,ldapalias,email, decode(trim(firstname || ' ' || lastname),'', ldapalias, (firstname || ' ' || lastname)) FULLNAME,status as USERSTATUS from demo_autosuggest where rownum < 999999999999" 
        jdbc_fetch_size => "100000" 
        schedule => "0 12 * * *"
    }
}
output{
    elasticsearch {
        hosts => ["http://***.***.119.199:9200"]
        index => "index_two"
        document_id =>"%{autosuggestid}"
        user => "elastic"
        password => "passwordes"
    }
}

标签： elasticsearchlogstashnssm

Answer 1

This is how I have configured the conf file of logstash to ingest data from two different sql statements into multiple Elasticsearch indices using logstash.


    input { 
        jdbc { 
            type=>"autosuggest"
            jdbc_driver_library => ""
            jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver" 
            jdbc_connection_string => "jdbc:oracle:thin:@nl0123vca0020.xyz-nl0123.abc.com:1521/pintu1"
            jdbc_user =>"usersys2"
            jdbc_password => "password"
            statement => "select name, id, address , col_1, col_2, col_3 from  demo_table_1"
            schedule => "0 */2 * * *"
            last_run_metadata_path => "E:/logstash-7.4.2/config/intouch_db_index_increment.txt"
            use_column_value => true
            tracking_column => "version"
        } 
jdbc {
        type=>"dbindex"
        jdbc_driver_library => ""
        jdbc_driver_class => "Java::oracle.jdbc.driver.OracleDriver"
        jdbc_connection_string => "jdbc:oracle:thin:@nl0123vca0020.xyz-nl0123.abc.com:1521/pintu1"
        jdbc_user =>"usersys2"
        jdbc_password => "password"
        statement  => "select autosuggestid,userid,ldapalias,email, decode(trim(firstname || ' ' || lastname),'', ldapalias, (firstname || ' ' || lastname)) FULLNAME,status as USERSTATUS from demo_autosuggest where rownum < 999999999999" 
        jdbc_fetch_size => "100000" 
        schedule => "0 12 * * *"
    }
    }

    filter {
       if [type] == "dbindex" 
        mutate {
            convert => {
                "contentid" => "string"
            }
        }
    }
    output{
        if [type] == "autosuggest"
    {  
        elasticsearch {
            hosts => ["http://***.***.119.199:9200"]
            index => "index_two"
            document_id =>"%{contentid}"
            user => "elastic" 
            password => "passwordes" 
        }
    }
   if [type] == "dbindex" 
    {
   elasticsearch {
        hosts => ["http://***.***.119.199:9200"]
        index => "index_two"
        document_id =>"%{autosuggestid}"
        user => "elastic"
        password => "passwordes"
    }
    } 
    }