Merge pull request #471 from metafacture/filemapColumns

Optionally set key and/or value column.

Author: blackwinter

.editorconfig

@@ -34,7 +34,7 @@ indent_size = 2
 [metafacture-io/src/test/resources/org/metafacture/io/compressed.txt]
 insert_final_newline = false
 
-[metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt]
+[metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test*.txt]
 trim_trailing_whitespace = false
 
 [metafacture-runner/src/main/dist/config/java-options.conf]

metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java

@@ -43,14 +43,19 @@
  *
  * The default {@link #setEncoding encoding} is UTF-8.
  * The default {@link #setSeparator separator} is {@code \t}.
+ * The default {@link #setKeyColumn keyColumn} is {@code 0}.
+ * The default {@link #setValueColumn valueColumn} is {@code 1}.
  *
- * By setting {@link #allowEmptyValues} to {@code true} the values in the
- * {@link Map} can be empty thus enabling e.g.
+ * <p>By setting {@link #setAllowEmptyValues allowEmptyValues} to {@code true},
+ * the values in the {@link Map} can be empty; thus enabling e.g.
  * {@link org.metafacture.metamorph.functions.SetReplace} to remove matching
  * keys.
  *
- * <strong>Important:</strong> All other lines that are not split in two parts
- * by the separator are ignored!
+ * <p>By setting {@link #setExpectedColumns expectedColumns} to
+ * {@code -1}, the number of columns is not checked.
+ *
+ * <p><strong>Important:</strong> Otherwise, all lines that are not split into
+ * the expected number of parts by the separator are ignored!
  *
  * @author Markus Michael Geipel
  */
@@ -59,10 +64,13 @@ public final class FileMap extends AbstractReadOnlyMap<String, String> {
     private final FileOpener fileOpener = new FileOpener();
     private final Map<String, String> map = new HashMap<>();
 
+    private ArrayList<String> filenames = new ArrayList<>();
     private Pattern split = Pattern.compile("\t", Pattern.LITERAL);
     private boolean allowEmptyValues;
     private boolean isUninitialized = true;
-    private ArrayList<String> filenames = new ArrayList<>();
+    private int expectedColumns;
+    private int keyColumn;
+    private int valueColumn = 1;
 
     /**
      * Creates an instance of {@link FileMap}.
@@ -79,14 +87,28 @@ private void init() {
      * Sets whether to allow empty values in the {@link Map} or ignore these
      * entries.
      *
-     * <strong>Default value: false </strong>
+     * <strong>Default value: false</strong>
      *
      * @param allowEmptyValues true if empty values in the Map are allowed
      */
     public void setAllowEmptyValues(final boolean allowEmptyValues) {
         this.allowEmptyValues = allowEmptyValues;
     }
 
+    /**
+     * Sets number of expected columns; lines with different number of columns
+     * are ignored. Set to {@code -1} to disable the check and allow arbitrary
+     * number of columns.
+     *
+     * <strong>Default value: calculated from {@link #setKeyColumn key} and
+     * {@link #setValueColumn value} columns</strong>
+     *
+     * @param expectedColumns number of expected columns
+     */
+    public void setExpectedColumns(final int expectedColumns) {
+        this.expectedColumns = expectedColumns;
+    }
+
     /**
      * Sets a comma separated list of files which provides the {@link Map}.
      *
@@ -141,14 +163,22 @@ private void loadFile(final String file) {
                 Reader reader = fileOpener.open(stream);
                 BufferedReader br = new BufferedReader(reader)
         ) {
+            final int minColumns = Math.max(keyColumn, valueColumn) + 1;
+            final int expColumns = expectedColumns != 0 ? expectedColumns : minColumns;
+
             String line;
             while ((line = br.readLine()) != null) {
                 if (line.isEmpty()) {
                     continue;
                 }
+
                 final String[] parts = allowEmptyValues ? split.split(line, -1) : split.split(line);
-                if (parts.length == 2) {
-                    map.put(parts[0], parts[1]);
+                if (parts.length < minColumns) {
+                    continue;
+                }
+
+                if (expColumns < 0 || parts.length == expColumns) {
+                    map.put(parts[keyColumn], parts[valueColumn]);
                 }
             }
         }
@@ -205,6 +235,28 @@ public void setSeparator(final String delimiter) {
         split = Pattern.compile(delimiter, Pattern.LITERAL);
     }
 
+    /**
+     * Sets the key column (0-based).
+     *
+     * <strong>Default value: {@code 0}</strong>
+     *
+     * @param keyColumn the key column
+     */
+    public void setKeyColumn(final int keyColumn) {
+        this.keyColumn = keyColumn;
+    }
+
+    /**
+     * Sets the value column (0-based).
+     *
+     * <strong>Default value: {@code 1}</strong>
+     *
+     * @param valueColumn the value column
+     */
+    public void setValueColumn(final int valueColumn) {
+        this.valueColumn = valueColumn;
+    }
+
     @Override
     public String get(final Object key) {
         if (isUninitialized) {

metamorph/src/main/resources/schemata/metamorph.xsd

@@ -629,6 +629,37 @@
                         The default separator is the tabulator. </documentation>
                 </annotation>
             </attribute>
+            <attribute name="keyColumn" use="optional" default="0">
+                <annotation>
+                    <documentation>Sets the key column (0-based).</documentation>
+                </annotation>
+                <simpleType>
+                    <restriction base="int">
+                        <minInclusive value="0" />
+                    </restriction>
+                </simpleType>
+            </attribute>
+            <attribute name="valueColumn" use="optional" default="1">
+                <annotation>
+                    <documentation>Sets the value column (0-based).</documentation>
+                </annotation>
+                <simpleType>
+                    <restriction base="int">
+                        <minInclusive value="0" />
+                    </restriction>
+                </simpleType>
+            </attribute>
+            <attribute name="expectedColumns" use="optional" default="0">
+                <annotation>
+                    <documentation>Sets number of expected columns; set to -1 to disable
+                        column check.</documentation>
+                </annotation>
+                <simpleType>
+                    <restriction base="int">
+                        <minInclusive value="-1" />
+                    </restriction>
+                </simpleType>
+            </attribute>
             <attribute ref="xml:base" />
         </complexType>
     </element>

metamorph/src/test/java/org/metafacture/metamorph/maps/FileMapTest.java

@@ -18,13 +18,16 @@
 
 import static org.metafacture.metamorph.TestHelpers.assertMorph;
 
+import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.metafacture.framework.StreamReceiver;
 import org.mockito.Mock;
 import org.mockito.junit.MockitoJUnit;
 import org.mockito.junit.MockitoRule;
 
+import java.util.function.Consumer;
+
 /**
  * Tests for class {@link FileMap}.
  *
@@ -39,14 +42,16 @@ public final class FileMapTest {
     @Mock
     private StreamReceiver receiver;
 
+    private static String MAPS = "org/metafacture/metamorph/maps/";
+
     private static String MORPH =
         "<rules>" +
         "  <data source='1'>" +
         "    <%s='map1' />" +
         "  </data>" +
         "</rules>" +
         "<maps>" +
-        "  <filemap name='map1' files='org/metafacture/metamorph/maps/%s' %s/>" +
+        "  <filemap name='map1' files='" + MAPS + "%s' %s/>" +
         "</maps>";
 
     @Test
@@ -67,6 +72,24 @@ public void shouldLookupValuesInFileBasedMap() {
         );
     }
 
+    @Test
+    public void shouldLookupValuesInFileBasedMapWithColumnOptions() {
+        assertMorph(receiver, buildMorph("lookup in", "keyColumn=\"1\" valueColumn=\"0\" expectedColumns=\"2\""),
+                i -> {
+                    i.startRecord("1");
+                    i.literal("1", "Germany");
+                    i.literal("1", "Fiji");
+                    i.endRecord();
+                },
+                o -> {
+                    o.get().startRecord("1");
+                    o.get().literal("1", "gw");
+                    o.get().literal("1", "fj");
+                    o.get().endRecord();
+                }
+        );
+    }
+
     @Test
     public void shouldWhitelistValuesInFileBasedMap() {
         assertMorph(receiver, buildMorph("whitelist map", ""),
@@ -206,6 +229,121 @@ public void shouldLookupValuesInBlockedGzipFileMap() {
         );
     }
 
+    @Test
+    public void shouldLoadFile() {
+        assertMap(379, i -> {
+            Assert.assertEquals("Puerto Rico", i.get("pr"));
+            Assert.assertNull(i.get("zz"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithEmptyValues() {
+        assertMap(380, i -> {
+            i.setAllowEmptyValues(true);
+
+            Assert.assertEquals("Puerto Rico", i.get("pr"));
+            Assert.assertEquals("", i.get("zz"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithSeparator() {
+        assertMap(99, i -> {
+            i.setSeparator(" ");
+
+            Assert.assertNull(i.get("pp\tPapua"));
+            Assert.assertEquals("Rico", i.get("pr\tPuerto"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithKeyColumn() {
+        assertMap(21, i -> {
+            i.setSeparator(" ");
+            i.setKeyColumn(2);
+
+            Assert.assertEquals("New", i.get("Guinea"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithValueColumn() {
+        assertMap(24, i -> {
+            i.setSeparator(" ");
+            i.setValueColumn(2);
+
+            Assert.assertEquals("Guinea", i.get("pp\tPapua"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithKeyAndValueColumn() {
+        assertMap(66, i -> {
+            i.setSeparator(" ");
+            i.setKeyColumn(1);
+            i.setValueColumn(0);
+
+            Assert.assertEquals("pr\tPuerto", i.get("Rico"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithExpectedColumns() {
+        assertMap(24, i -> {
+            i.setSeparator(" ");
+            i.setExpectedColumns(3);
+
+            Assert.assertEquals("New", i.get("pp\tPapua"));
+        });
+    }
+
+    @Test
+    public void shouldLoadFileWithArbitraryExpectedColumns() {
+        assertMap(149, i -> {
+            i.setSeparator(" ");
+            i.setExpectedColumns(-1);
+
+            Assert.assertEquals("New", i.get("pp\tPapua"));
+        });
+    }
+
+    @Test
+    public void shouldNotLoadFileWithOutOfRangeKeyColumn() {
+        assertMap(0, i -> {
+            i.setKeyColumn(2);
+        });
+    }
+
+    @Test
+    public void shouldNotLoadFileWithOutOfRangeValueColumn() {
+        assertMap(0, i -> {
+            i.setValueColumn(2);
+        });
+    }
+
+    @Test
+    public void shouldNotLoadFileWithTooFewExpectedColumns() {
+        assertMap(0, i -> {
+            i.setExpectedColumns(1);
+        });
+    }
+
+    @Test
+    public void shouldNotLoadFileWithTooManyExpectedColumns() {
+        assertMap(0, i -> {
+            i.setExpectedColumns(99);
+        });
+    }
+
+    private void assertMap(final int size, final Consumer<FileMap> consumer) {
+        final FileMap fileMap = new FileMap();
+        fileMap.setFile(MAPS + "file-map-test-columns.txt");
+
+        consumer.accept(fileMap);
+        Assert.assertEquals(size, fileMap.keySet().size());
+    }
+
     private String buildMorph(final String data, final String options) {
         return buildMorph(data, "file-map-test.txt", options);
     }