java - Elasticsearch Performance Extreme 在非常长的字符串字段下

Question

try {
    Settings settings = Settings.builder().put("cluster.name", "phm").build();
    TransportClient client = new PreBuiltTransportClient(settings)
            .addTransportAddress(new TransportAddress(InetAddress.getByName("..."), 9300))
            .addTransportAddress(new TransportAddress(InetAddress.getByName("..."), 9300))
            .addTransportAddress(new TransportAddress(InetAddress.getByName("..."), 9300));
    client.admin().indices().create(new CreateIndexRequest("newtest")).actionGet();
    System.out.println("Step1");

    PutMappingResponse putMappingResponse = client.admin().indices()
            .preparePutMapping("newtest")
            .setType("doc")
            .setSource(jsonBuilder().prettyPrint()
                .startObject()
                        .startObject("properties")
                    .startObject("rawdata").field("type", "keyword").field("index", "false").endObject()
                    .startObject("spectrum").field("type", "keyword").field("index", "false").endObject()
                    .startObject("id").field("type", "integer").endObject()
                     .startObject("timestamp").field("type", "integer").endObject()
                    .startObject("health").field("type", "integer").endObject()
                    .startObject("rul").field("type", "integer").endObject()
                    .startObject("RMS").field("type", "integer").endObject()
                    .startObject("VAR").field("type", "integer").endObject()
                    .startObject("peak").field("type", "integer").endObject()
                    .startObject("CrestIndex").field("type", "integer").endObject()
                    .startObject("peakpeak").field("type", "integer").endObject()
                    .startObject("MarginIndex").field("type", "integer").endObject()
                    .startObject("SkewNess").field("type", "integer").endObject()
                    .startObject("SkewnessIndex").field("type", "integer").endObject()
                    .startObject("kurtosis").field("type", "integer").endObject()
                    .startObject("KurtosisIndex").field("type", "integer").endObject()
                    .startObject("InpulseIndex").field("type", "integer").endObject()
                    .startObject("WaveformIndex").field("type", "integer").endObject()
                            .endObject()
                    .endObject())
            .execute().actionGet();

    System.out.println("Step2");






    String raw_data = "";
    String spectrum = "";
    for (int i = 0; i < 100000; i++) {
        raw_data = raw_data + "aaaaaaaaaa";
        System.out.println(i);
    }
    for (int i = 0; i < 50000; i++) {
        spectrum = spectrum + "bbbbbbbbbb";

    }


    for (int j = 0; j < BULK_NUM; j++) {
        BulkRequestBuilder request = client.prepareBulk();
        for (int i = 0; i < BULK_SIZE; i++) {
            Map<String, Object> parseObject = new HashMap<String, Object>();
            int Max = 10000;
            int Min = 0;
            parseObject.put("id", 10000 * j + i);
            parseObject.put("timestamp", timestamp);
            parseObject.put("health", health);
            parseObject.put("rul", rul);
            parseObject.put("RMS", RMS);
            parseObject.put("VAR", VAR);
            parseObject.put("peak", peak);
            parseObject.put("CrestIndex", CrestIndex);
            parseObject.put("peakpeak", peakpeak);
            parseObject.put("MarginIndex", MarginIndex);
            parseObject.put("SkewNess", SkewNess);
            parseObject.put("SkewnessIndex", SkewnessIndex);
            parseObject.put("kurtosis", kurtosis);
            parseObject.put("KurtosisIndex", KurtosisIndex);
            parseObject.put("InpulseIndex", InpulseIndex);
            parseObject.put("WaveformIndex", WaveformIndex);
            parseObject.put("RawData", raw_data);
            parseObject.put("Spectrum", spectrum);

            request.add(new IndexRequest("newtest", "doc")
                    .source(parseObject));
        }

         BulkResponse bulkResponse = request.execute().get();

        System.out.println(j);

    }
        client.close();



}
catch (Exception e){
    System.out.println("cluster error!");
    exit(2);
}

大家好，我遇到了插入慢的问题。更具体地说，我要插入的数据包括 100,0000 个长度的字符串（详见代码）。我使用了 5 的 bulksize（更多会导致内存问题）。我用“index：no”制作了那些长字符串字段。但是，当我在 3 节点集群（intel Xeon 4 core，HDD，16GB）上测试我的插入时，速度大约是每秒 30-50 个这样的数据，这很慢。我在网上搜索并知道有很多配置或设置我可以修改，但我很好奇根据目前的情况，估计的最大插入速度是多少，比如是否有可能将其提高到每秒 10k 或由于某些东西的瓶颈，这根本不可能。太感谢了。我的公司正在处理这个问题，我完全不知道..

标签： javadatabaseperformanceelasticsearchcomputer-science