Skip to content
This repository was archived by the owner on Mar 21, 2023. It is now read-only.

Commit f00cf72

Browse files
joschijoschi
authored
Properly handle empty or missing templates in NetFlow 9 (#17)
Fixes #16
1 parent 4b0a62e commit f00cf72

File tree

8 files changed

+92
-10
lines changed

8 files changed

+92
-10
lines changed

‎src/main/java/org/graylog/plugins/netflow/codecs/NetFlowCodec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public Collection<Message> decodeMessages(@Nonnull RawMessage rawMessage) {
106106
try {
107107
return NetFlowParser.parse(rawMessage, templateCache, typeRegistry);
108108
} catch (FlowException e) {
109-
LOG.error("Error parsing NetFlow packet", e);
109+
LOG.error("Error parsing NetFlow packet <{}> received from <{}>", rawMessage.getId(), rawMessage.getRemoteAddress(), e);
110110
return null;
111111
}
112112
}

‎src/main/java/org/graylog/plugins/netflow/flows/EmptyTemplateException.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,7 @@
2222
package org.graylog.plugins.netflow.flows;
2323

2424
public class EmptyTemplateException extends FlowException {
25+
public EmptyTemplateException(String message) {
26+
super(message);
27+
}
2528
}

‎src/main/java/org/graylog/plugins/netflow/v9/NetFlowV9Parser.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.google.common.collect.ImmutableList;
1919
import com.google.common.collect.ImmutableMap;
2020
import io.netty.buffer.ByteBuf;
21+
import org.graylog.plugins.netflow.flows.EmptyTemplateException;
2122
import org.graylog.plugins.netflow.flows.InvalidFlowVersionException;
2223

2324
import java.util.ArrayList;
@@ -28,13 +29,13 @@
2829

2930

3031
public class NetFlowV9Parser {
31-
private static AtomicReference<NetFlowV9OptionTemplate> optionTemplateReference = new AtomicReference<>();
32+
private static final AtomicReference<NetFlowV9OptionTemplate> optionTemplateReference = new AtomicReference<>();
3233

3334
public static NetFlowV9Packet parsePacket(ByteBuf bb, NetFlowV9TemplateCache cache, NetFlowV9FieldTypeRegistry typeRegistry) {
3435
final int dataLength = bb.readableBytes();
3536
final NetFlowV9Header header = parseHeader(bb);
3637

37-
final ImmutableList.Builder<NetFlowV9Template> allTemplates = ImmutableList.builder();
38+
final List<NetFlowV9Template> allTemplates = new ArrayList<>();
3839
NetFlowV9OptionTemplate optTemplate = null;
3940
List<NetFlowV9BaseRecord> records = Collections.emptyList();
4041
while (bb.isReadable()) {
@@ -51,13 +52,16 @@ public static NetFlowV9Packet parsePacket(ByteBuf bb, NetFlowV9TemplateCache cac
5152
optionTemplateReference.set(optTemplate);
5253
} else {
5354
bb.resetReaderIndex();
55+
if (cache.isEmpty()) {
56+
throw new EmptyTemplateException("Unable to parse NetFlow 9 records without template. Discarding packet.");
57+
}
5458
records = parseRecords(bb, cache);
5559
}
5660
}
5761

5862
return NetFlowV9Packet.create(
5963
header,
60-
allTemplates.build(),
64+
allTemplates,
6165
optTemplate,
6266
records,
6367
dataLength);

‎src/main/java/org/graylog/plugins/netflow/v9/NetFlowV9TemplateCache.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ public NetFlowV9Template get(int id) {
8787
return cache.getIfPresent(id);
8888
}
8989

90+
public boolean isEmpty() {
91+
return cache.size() == 0L;
92+
}
93+
9094
@Override
9195
public void run() {
9296
if (cache.size() != 0) {

‎src/test/java/org/graylog/plugins/netflow/codecs/NetFlowCodecTest.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,14 @@ public void decodeMessagesSuccessfullyDecodesNetFlowV9() throws Exception {
221221
.containsEntry("nf_snmp_output", 0);
222222
}
223223

224+
@Test
225+
public void decodeMessagesThrowsEmptyTemplateExceptionWithIncompleteNetFlowV9() throws Exception {
226+
final byte[] b = Resources.toByteArray(Resources.getResource("netflow-data/netflow-v9-3_incomplete.dat"));
227+
final InetSocketAddress source = new InetSocketAddress(InetAddress.getLocalHost(), 12345);
228+
229+
assertThat(codec.decodeMessages(new RawMessage(b, source))).isNull();
230+
}
231+
224232
@Test
225233
public void pcap_softflowd_NetFlowV5() throws Exception {
226234
final List<Message> allMessages = new ArrayList<>();
@@ -381,6 +389,29 @@ public void pcap_nprobe_NetFlowV9_2() throws Exception {
381389
}
382390
assertThat(allMessages)
383391
.hasSize(6)
384-
.allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(9));
392+
.allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(9));
393+
}
394+
395+
@Test
396+
public void pcap_nprobe_NetFlowV9_4() throws Exception {
397+
final List<Message> allMessages = new ArrayList<>();
398+
try (InputStream inputStream = Resources.getResource("netflow-data/nprobe-netflow9-4.pcap").openStream()) {
399+
final Pcap pcap = Pcap.openStream(inputStream);
400+
pcap.loop(packet -> {
401+
if (packet.hasProtocol(Protocol.UDP)) {
402+
final UDPPacket udp = (UDPPacket) packet.getPacket(Protocol.UDP);
403+
final InetSocketAddress source = new InetSocketAddress(udp.getSourceIP(), udp.getSourcePort());
404+
final Collection<Message> messages = codec.decodeMessages(new RawMessage(udp.getPayload().getArray(), source));
405+
if (messages != null) {
406+
allMessages.addAll(messages);
407+
}
408+
}
409+
return true;
410+
}
411+
);
412+
}
413+
assertThat(allMessages)
414+
.hasSize(1)
415+
.allSatisfy(message -> assertThat(message.getField("nf_version")).isEqualTo(9));
385416
}
386417
}

‎src/test/java/org/graylog/plugins/netflow/v9/NetFlowV9ParserTest.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import io.pkts.Pcap;
2525
import io.pkts.packet.UDPPacket;
2626
import io.pkts.protocol.Protocol;
27+
import org.graylog.plugins.netflow.flows.EmptyTemplateException;
2728
import org.graylog2.shared.bindings.providers.ObjectMapperProvider;
2829
import org.junit.Before;
2930
import org.junit.Rule;
@@ -38,6 +39,7 @@
3839
import java.util.concurrent.Executors;
3940

4041
import static org.assertj.core.api.Assertions.assertThat;
42+
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
4143
import static org.junit.Assert.assertEquals;
4244
import static org.junit.Assert.assertNotNull;
4345

@@ -124,6 +126,13 @@ public void testParse() throws IOException {
124126
assertEquals(1, p3.records().size());
125127
}
126128

129+
@Test
130+
public void testParseIncomplete() throws Exception {
131+
final byte[] b = Resources.toByteArray(Resources.getResource("netflow-data/netflow-v9-3_incomplete.dat"));
132+
assertThatExceptionOfType(EmptyTemplateException.class)
133+
.isThrownBy(() -> NetFlowV9Parser.parsePacket(Unpooled.wrappedBuffer(b), cache, typeRegistry));
134+
}
135+
127136
@Test
128137
public void pcap_softflowd_NetFlowV9() throws Exception {
129138
final List<NetFlowV9BaseRecord> allRecords = new ArrayList<>();
@@ -428,10 +437,14 @@ public void pcap_nprobe_NetFlowV9_4() throws Exception {
428437
if (packet.hasProtocol(Protocol.UDP)) {
429438
final UDPPacket udp = (UDPPacket) packet.getPacket(Protocol.UDP);
430439
final ByteBuf byteBuf = Unpooled.wrappedBuffer(udp.getPayload().getArray());
431-
final NetFlowV9Packet netFlowV9Packet = NetFlowV9Parser.parsePacket(byteBuf, cache, typeRegistry);
432-
assertThat(netFlowV9Packet).isNotNull();
433-
allTemplates.addAll(netFlowV9Packet.templates());
434-
allRecords.addAll(netFlowV9Packet.records());
440+
try {
441+
final NetFlowV9Packet netFlowV9Packet = NetFlowV9Parser.parsePacket(byteBuf, cache, typeRegistry);
442+
assertThat(netFlowV9Packet).isNotNull();
443+
allTemplates.addAll(netFlowV9Packet.templates());
444+
allRecords.addAll(netFlowV9Packet.records());
445+
} catch (EmptyTemplateException e) {
446+
// ignore
447+
}
435448
}
436449
return true;
437450
}
@@ -461,7 +474,7 @@ public void pcap_nprobe_NetFlowV9_4() throws Exception {
461474
).build()
462475
)
463476
);
464-
assertThat(allRecords).hasSize(898);
477+
assertThat(allRecords).hasSize(2);
465478
}
466479

467480
private String name(NetFlowV9FieldDef def) {

‎src/test/java/org/graylog/plugins/netflow/v9/NetFlowV9TemplateCacheTest.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,4 +178,31 @@ public void loadsCacheFileOnStart() throws Exception {
178178
assertThat(templateCache.get(0)).isEqualTo(template1);
179179
assertThat(templateCache.get(1)).isEqualTo(template2);
180180
}
181+
182+
@Test
183+
public void isEmptyReturnsFalseForNonEmptyCache() throws Exception {
184+
final byte[] json = ("{" +
185+
"\"0\":{" +
186+
"\"template_id\":0," +
187+
"\"field_count\":1," +
188+
"\"definitions\":[{\"type\":{\"id\":0,\"value_type\":\"UINT64\",\"name\":\"foobar\"},\"length\":8}]" +
189+
"}," +
190+
"\"1\":{" +
191+
"\"template_id\":1," +
192+
"\"field_count\":1," +
193+
"\"definitions\":[{\"type\":{\"id\":0,\"value_type\":\"IPV4\",\"name\":\"covfefe\"},\"length\":4}]}}")
194+
.getBytes(StandardCharsets.UTF_8);
195+
assertThat(Files.write(cachePath, json)).isEqualTo(cachePath);
196+
assertThat(Files.size(cachePath)).isEqualTo(json.length);
197+
198+
final NetFlowV9TemplateCache templateCache = new NetFlowV9TemplateCache(100L, cachePath, 300, executorService, objectMapper);
199+
200+
assertThat(templateCache.isEmpty()).isFalse();
201+
}
202+
203+
@Test
204+
public void isEmptyReturnsTrueForNonEmptyCache() throws Exception {
205+
final NetFlowV9TemplateCache templateCache = new NetFlowV9TemplateCache(100L, cachePath, 300, executorService, objectMapper);
206+
assertThat(templateCache.isEmpty()).isTrue();
207+
}
181208
}

‎src/test/resources/netflow-data/netflow-v9-3_incomplete.dat‎

80 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)