Add ability to exclude sample results to be sent as logs based on response code regex (#48)

* Add ability to send only errors as logs

* isErrorResponse to handle non-HTTP response code

* Reverted IDEA default formatting

* Add ability to exclude sample results to be sent as logs based on response code regex

* Added testExcludeLogsResponseCodeRegexMatching

* Slightly updated README to highlight exclude strategy

* Added testExcludeLogsResponseCodeRegexDefaultEmpty

* Revert default import order

* Update README.md

Co-authored-by: Alex Lopez <alex.lopez.zorzano@gmail.com>

---------

Co-authored-by: Alex Lopez <alex.lopez.zorzano@gmail.com>

Author: arustamov

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changes
 
+## 0.5.0
+* [Added] Add ability to exclude sample results to be sent as logs based on response code regex
+  See [#47](https://github.com/DataDog/jmeter-datadog-backend-listener/issues/47)
+
 ## 0.4.0
 * [Changed] Set configured tags on plugin generated logs. (See [#45](https://github.com/DataDog/jmeter-datadog-backend-listener/pull/45)).
 

README.md

@@ -46,6 +46,7 @@ The plugin has the following configuration options:
 |logsBatchSize|false|500|Logs are submitted in batches of size `logsBatchSize` as soon as this size is reached.|
 |sendResultsAsLogs|false|false|By default only metrics are reported to Datadog. To report individual test results as log events, set this field to `true`.|
 |includeSubresults|false|false|A subresult is for instance when an individual HTTP request has to follow redirects. By default subresults are ignored.|
+|excludeLogsResponseCodeRegex|false|`""`| Setting `sendResultsAsLogs` will submit all results as logs to Datadog by default. This option lets you exclude results whose response code matches a given regex. For example, you may set this option to `[123][0-5][0-9]` to only submit errors.|
 |customTags|false|`""`|Comma-separated list of tags to add to every metric
 
 ## Troubleshooting

pom.xml

@@ -7,7 +7,7 @@
   <groupId>com.datadoghq</groupId>
   <artifactId>jmeter-datadog-backend-listener</artifactId>
 
-  <version>0.4.0</version>
+  <version>0.5.0</version>
   <name>jmeter-datadog-backend-listener</name>
   <url>https://github.com/DataDog/jmeter-datadog-backend-listener</url>
   <description>Datadog JMeter plugin</description>

src/main/java/org/datadog/jmeter/plugins/DatadogBackendClient.java

@@ -5,16 +5,15 @@
 
 package org.datadog.jmeter.plugins;
 
-import java.util.Arrays;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import net.minidev.json.JSONObject;
 import org.apache.jmeter.config.Arguments;
@@ -175,10 +174,12 @@ private void extractData(SampleResult sampleResult) {
         userMetrics.add(sampleResult);
         this.extractMetrics(sampleResult);
         if(configuration.shouldSendResultsAsLogs()) {
-            this.extractLogs(sampleResult);
-            if(logsBuffer.size() >= configuration.getLogsBatchSize()) {
-                datadogClient.submitLogs(logsBuffer, this.configuration.getCustomTags());
-                logsBuffer.clear();
+            if(!shouldExcludeSampleResultAsLogs(sampleResult)) {
+                this.extractLogs(sampleResult);
+                if (logsBuffer.size() >= configuration.getLogsBatchSize()) {
+                    datadogClient.submitLogs(logsBuffer, this.configuration.getCustomTags());
+                    logsBuffer.clear();
+                }
             }
         }
         if(configuration.shouldIncludeSubResults()) {
@@ -188,6 +189,14 @@ private void extractData(SampleResult sampleResult) {
         }
     }
 
+    /**
+     * Called for each individual result. It checks if logs for the sample result should be excluded
+     * @param sampleResult the result
+     */
+    private boolean shouldExcludeSampleResultAsLogs(SampleResult sampleResult) {
+        return configuration.getExcludeLogsResponseCodeRegex().matcher(sampleResult.getResponseCode()).matches();
+    }
+
     /**
      * Called for each individual result. It extracts metrics and give them to the {@link ConcurrentAggregator} instance for aggregation.
      * @param sampleResult the result

src/main/java/org/datadog/jmeter/plugins/DatadogConfiguration.java

@@ -49,6 +49,12 @@ public class DatadogConfiguration {
      */
     private boolean sendResultsAsLogs;
 
+    /**
+     * User configurable. This options configures which Datadog logs to exclude from submission using a regex
+     * that matches on the response code.
+     */
+    private Pattern excludeLogsResponseCodeRegex = null;
+
     /**
      * This options configures whether or not to collect metrics and logs for jmeter subresults.
      */
@@ -74,6 +80,7 @@ public class DatadogConfiguration {
     private static final String LOGS_BATCH_SIZE = "logsBatchSize";
     private static final String SEND_RESULTS_AS_LOGS = "sendResultsAsLogs";
     private static final String INCLUDE_SUB_RESULTS = "includeSubresults";
+    private static final String EXCLUDE_LOGS_RESPONSE_CODE_REGEX = "excludeLogsResponseCodeRegex";
     private static final String SAMPLERS_REGEX = "samplersRegex";
     private static final String CUSTOM_TAGS ="customTags";
 
@@ -84,6 +91,7 @@ public class DatadogConfiguration {
     private static final int DEFAULT_LOGS_BATCH_SIZE = 500;
     private static final boolean DEFAULT_SEND_RESULTS_AS_LOGS = true;
     private static final boolean DEFAULT_INCLUDE_SUB_RESULTS = false;
+    private static final String DEFAULT_EXCLUDE_LOGS_RESPONSE_CODE_REGEX = "";
     private static final String DEFAULT_SAMPLERS_REGEX = "";
     private static final String DEFAULT_CUSTOM_TAGS = "";
 
@@ -98,6 +106,7 @@ public static Arguments getPluginArguments() {
         arguments.addArgument(LOGS_BATCH_SIZE, String.valueOf(DEFAULT_LOGS_BATCH_SIZE));
         arguments.addArgument(SEND_RESULTS_AS_LOGS, String.valueOf(DEFAULT_SEND_RESULTS_AS_LOGS));
         arguments.addArgument(INCLUDE_SUB_RESULTS, String.valueOf(DEFAULT_INCLUDE_SUB_RESULTS));
+        arguments.addArgument(EXCLUDE_LOGS_RESPONSE_CODE_REGEX, DEFAULT_EXCLUDE_LOGS_RESPONSE_CODE_REGEX);
         arguments.addArgument(SAMPLERS_REGEX, DEFAULT_SAMPLERS_REGEX);
         arguments.addArgument(CUSTOM_TAGS, DEFAULT_CUSTOM_TAGS);
         return arguments;
@@ -141,8 +150,11 @@ public static DatadogConfiguration parseConfiguration(BackendListenerContext con
             throw new DatadogConfigurationException("Invalid 'includeSubResults'. Value '" + includeSubResults + "' is not a boolean.");
         }
         configuration.includeSubResults = Boolean.parseBoolean(includeSubResults);
+
         configuration.samplersRegex = Pattern.compile(context.getParameter(SAMPLERS_REGEX, DEFAULT_SAMPLERS_REGEX));
 
+        configuration.excludeLogsResponseCodeRegex = Pattern.compile(context.getParameter(EXCLUDE_LOGS_RESPONSE_CODE_REGEX, DEFAULT_EXCLUDE_LOGS_RESPONSE_CODE_REGEX));
+
         String customTagsString = context.getParameter(CUSTOM_TAGS, String.valueOf(DEFAULT_CUSTOM_TAGS));
         List<String> customTags = new ArrayList<>();
         if(customTagsString.contains(",")){
@@ -186,6 +198,10 @@ public boolean shouldIncludeSubResults() {
         return includeSubResults;
     }
 
+    public Pattern getExcludeLogsResponseCodeRegex() {
+        return excludeLogsResponseCodeRegex;
+    }
+
     public Pattern getSamplersRegex() {
         return samplersRegex;
     }

src/test/java/org/datadog/jmeter/plugins/DatadogBackendClientTest.java

@@ -51,6 +51,7 @@ public class DatadogBackendClientTest
             put("logsBatchSize", "0");
             put("sendResultsAsLogs", "true");
             put("includeSubresults", "false");
+            put("excludeLogsResponseCodeRegex", "");
             put("samplersRegex", "^foo\\d*$");
             put("customTags", "key:value");
         }
@@ -85,9 +86,13 @@ public void teardownMocks() throws Exception {
 
 
     private SampleResult createDummySampleResult(String sampleLabel) {
+        return createDummySampleResult(sampleLabel, "123");
+    }
+
+    private SampleResult createDummySampleResult(String sampleLabel, String responseCode) {
         SampleResult result = SampleResult.createTestSample(1, 126);
         result.setSuccessful(true);
-        result.setResponseCode("123");
+        result.setResponseCode(responseCode);
         result.setSampleLabel(sampleLabel);
         result.setSampleCount(10);
         result.setErrorCount(1);
@@ -268,7 +273,7 @@ private void assertMetricsWithTag(List<DatadogMetric> metrics, Map<String, Doubl
     }
 
     @Test
-    public void testRegexNotMatching() {
+    public void testSamplersRegexNotMatching() {
         SampleResult result1 = createDummySampleResult("foo1");
         SampleResult resultA = createDummySampleResult("fooA");
 
@@ -281,4 +286,38 @@ public void testRegexNotMatching() {
         Assert.assertEquals("foo1", this.logsBuffer.get(0).getAsString("sample_label"));
     }
 
+    @Test
+    public void testExcludeLogsResponseCodeRegexDefaultEmpty() {
+        SampleResult result1 = createDummySampleResult("foo1", "200");
+        SampleResult result2 = createDummySampleResult("foo2", "301");
+        SampleResult result3 = createDummySampleResult("foo3", "404");
+        SampleResult result4 = createDummySampleResult("foo4", "Non HTTP response code: java.net.NoRouteToHostException");
+
+        this.client.handleSampleResults(Arrays.asList(result1, result2, result3, result4), context);
+        Assert.assertEquals(4, this.logsBuffer.size());
+        Assert.assertEquals("foo1", this.logsBuffer.get(0).getAsString("sample_label"));
+        Assert.assertEquals("foo2", this.logsBuffer.get(1).getAsString("sample_label"));
+        Assert.assertEquals("foo3", this.logsBuffer.get(2).getAsString("sample_label"));
+        Assert.assertEquals("foo4", this.logsBuffer.get(3).getAsString("sample_label"));
+    }
+
+    @Test
+    public void testExcludeLogsResponseCodeRegexMatching() throws Exception {
+        HashMap<String, String> config = new HashMap<>(DEFAULT_VALID_TEST_CONFIG);
+        config.put("excludeLogsResponseCodeRegex", "^[23][0-5][0-9]$");
+        DatadogBackendClient client = new DatadogBackendClient();
+        BackendListenerContext context = new BackendListenerContext(config);
+        client.setupTest(context);
+
+        SampleResult result1 = createDummySampleResult("foo1", "200");
+        SampleResult result2 = createDummySampleResult("foo2", "301");
+        SampleResult result3 = createDummySampleResult("foo3", "404");
+        SampleResult result4 = createDummySampleResult("foo4", "Non HTTP response code: java.net.NoRouteToHostException");
+
+        client.handleSampleResults(Arrays.asList(result1, result2, result3, result4), context);
+        Assert.assertEquals(2, this.logsBuffer.size());
+        Assert.assertEquals("foo3", this.logsBuffer.get(0).getAsString("sample_label"));
+        Assert.assertEquals("foo4", this.logsBuffer.get(1).getAsString("sample_label"));
+    }
+
 }

src/test/java/org/datadog/jmeter/plugins/DatadogConfigurationTest.java

@@ -5,16 +5,16 @@
 
 package org.datadog.jmeter.plugins;
 
-import java.util.Arrays;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.regex.PatternSyntaxException;
 import org.apache.jmeter.config.Arguments;
 import org.apache.jmeter.visualizers.backend.BackendListenerContext;
 import org.datadog.jmeter.plugins.exceptions.DatadogConfigurationException;
-import org.junit.Test;
 import org.junit.Assert;
+import org.junit.Test;
 
 public class DatadogConfigurationTest {
     private static final String API_URL_PARAM = "datadogUrl";
@@ -24,13 +24,14 @@ public class DatadogConfigurationTest {
     private static final String LOGS_BATCH_SIZE = "logsBatchSize";
     private static final String SEND_RESULTS_AS_LOGS = "sendResultsAsLogs";
     private static final String INCLUDE_SUB_RESULTS = "includeSubresults";
+    private static final String EXCLUDE_LOGS_RESPONSE_CODE_REGEX = "excludeLogsResponseCodeRegex";
     private static final String SAMPLERS_REGEX = "samplersRegex";
     private static final String CUSTOM_TAGS = "customTags";
 
     @Test
     public void testArguments(){
         Arguments args = DatadogConfiguration.getPluginArguments();
-        Assert.assertEquals(9, args.getArgumentCount());
+        Assert.assertEquals(10, args.getArgumentCount());
 
         Map<String, String> argumentsMap = args.getArgumentsAsMap();
         Assert.assertTrue(argumentsMap.containsKey(API_URL_PARAM));
@@ -40,6 +41,7 @@ public void testArguments(){
         Assert.assertTrue(argumentsMap.containsKey(LOGS_BATCH_SIZE));
         Assert.assertTrue(argumentsMap.containsKey(SEND_RESULTS_AS_LOGS));
         Assert.assertTrue(argumentsMap.containsKey(INCLUDE_SUB_RESULTS));
+        Assert.assertTrue(argumentsMap.containsKey(EXCLUDE_LOGS_RESPONSE_CODE_REGEX));
         Assert.assertTrue(argumentsMap.containsKey(SAMPLERS_REGEX));
         Assert.assertTrue(argumentsMap.containsKey(CUSTOM_TAGS));
     }
@@ -55,6 +57,7 @@ public void testValidConfiguration() throws DatadogConfigurationException {
                 put(LOGS_BATCH_SIZE, "11");
                 put(SEND_RESULTS_AS_LOGS, "true");
                 put(INCLUDE_SUB_RESULTS, "false");
+                put(EXCLUDE_LOGS_RESPONSE_CODE_REGEX, "");
                 put(SAMPLERS_REGEX, "false");
                 put(CUSTOM_TAGS, "key:value");
             }
@@ -134,7 +137,29 @@ public void testIncludeSubresultsNotBoolean() throws DatadogConfigurationExcepti
     }
 
     @Test(expected = PatternSyntaxException.class)
-    public void testInvalidRegex() throws DatadogConfigurationException {
+    public void testInvalidExcludeLogsResponseCodeRegex() throws DatadogConfigurationException {
+        Map<String, String> config = new HashMap<String, String>() {
+            {
+                put("apiKey", "123456");
+                put(EXCLUDE_LOGS_RESPONSE_CODE_REGEX, "[");
+            }
+        };
+        DatadogConfiguration.parseConfiguration(new BackendListenerContext(config));
+    }
+
+    @Test
+    public void testValidExcludeLogsResponseCodeRegex() throws DatadogConfigurationException {
+        Map<String, String> config = new HashMap<String, String>() {
+            {
+                put("apiKey", "123456");
+                put(EXCLUDE_LOGS_RESPONSE_CODE_REGEX, "[123][0-5][0-9]");
+            }
+        };
+        DatadogConfiguration.parseConfiguration(new BackendListenerContext(config));
+    }
+
+    @Test(expected = PatternSyntaxException.class)
+    public void testInvalidSamplersRegex() throws DatadogConfigurationException {
         Map<String, String> config = new HashMap<String, String>() {
             {
                 put("apiKey", "123456");
@@ -145,7 +170,7 @@ public void testInvalidRegex() throws DatadogConfigurationException {
     }
 
     @Test
-    public void testValidRegex() throws DatadogConfigurationException {
+    public void testValidSamplersRegex() throws DatadogConfigurationException {
         Map<String, String> config = new HashMap<String, String>() {
             {
                 put("apiKey", "123456");