From 8d2cb080cf3c9ee8f7b978a8a42baebcc0a3c34e Mon Sep 17 00:00:00 2001 From: Jens Wille Date: Fri, 24 Jun 2022 17:26:53 +0200 Subject: [PATCH 1/2] Add support for compressed files as `FileMap` input. Delegates opening the input stream to `FileOpener` with all its configurability. --- .../java/org/metafacture/io/FileOpener.java | 60 +++++++++----- metamorph/build.gradle | 1 + .../metafacture/metamorph/maps/FileMap.java | 44 ++++++++-- .../src/main/resources/schemata/metamorph.xsd | 15 ++++ .../metamorph/maps/FileMapTest.java | 76 ++++++++++++++++-- .../metamorph/maps/file-map-test.txt.bgzf | Bin 0 -> 16392 bytes .../metamorph/maps/file-map-test.txt.gz | Bin 0 -> 3088 bytes 7 files changed, 158 insertions(+), 38 deletions(-) create mode 100644 metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.bgzf create mode 100644 metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.gz diff --git a/metafacture-io/src/main/java/org/metafacture/io/FileOpener.java b/metafacture-io/src/main/java/org/metafacture/io/FileOpener.java index 2da6aa472..b88b9aabd 100644 --- a/metafacture-io/src/main/java/org/metafacture/io/FileOpener.java +++ b/metafacture-io/src/main/java/org/metafacture/io/FileOpener.java @@ -40,7 +40,7 @@ */ @Description("Opens a file.") @In(String.class) -@Out(java.io.Reader.class) +@Out(Reader.class) @FluxCommand("open-file") public final class FileOpener extends DefaultObjectPipe> { @@ -66,8 +66,7 @@ public String getEncoding() { /** * Sets the encoding used to open the resource. * - * @param encoding - * new encoding + * @param encoding new encoding */ public void setEncoding(final String encoding) { this.encoding = encoding; @@ -83,7 +82,7 @@ public FileCompression getCompression() { } /** - * * Sets the compression of the file. + * Sets the compression of the file. * * @param compression the {@link FileCompression} */ @@ -94,7 +93,7 @@ public void setCompression(final FileCompression compression) { /** * Sets the compression of the file. * - * @param compression the name of the compression. + * @param compression the name of the compression */ public void setCompression(final String compression) { setCompression(FileCompression.valueOf(compression.toUpperCase())); @@ -112,35 +111,52 @@ public boolean getDecompressConcatenated() { /** * Flags whether to use decompress concatenated file compression. * - * @param decompressConcatenated true if file compression should be decompresses - * concatenated + * @param decompressConcatenated true if file compression should decompress concatenated */ public void setDecompressConcatenated(final boolean decompressConcatenated) { this.decompressConcatenated = decompressConcatenated; } - @Override - public void process(final String file) { + /** + * Opens a file. + * + * @param file the file + * @return a Reader + * @throws IOException if an I/O error occurs + */ + public Reader open(final String file) throws IOException { + return open(new FileInputStream(file)); + } + + /** + * Opens a file stream. + * + * @param stream the stream + * @return a Reader + * @throws IOException if an I/O error occurs + */ + public Reader open(final InputStream stream) throws IOException { try { - final InputStream fileStream = new FileInputStream(file); + final InputStream decompressor = compression.createDecompressor(stream, decompressConcatenated); try { - final InputStream decompressor = compression.createDecompressor(fileStream, decompressConcatenated); - try { - - final Reader reader = new InputStreamReader(new BOMInputStream( - decompressor), encoding); - getReceiver().process(reader); - } - catch (final IOException | MetafactureException e) { - decompressor.close(); - throw e; - } + return new InputStreamReader(new BOMInputStream(decompressor), encoding); } catch (final IOException | MetafactureException e) { - fileStream.close(); + decompressor.close(); throw e; } } + catch (final IOException | MetafactureException e) { + stream.close(); + throw e; + } + } + + @Override + public void process(final String file) { + try { + getReceiver().process(open(file)); + } catch (final IOException e) { throw new MetafactureException(e); } diff --git a/metamorph/build.gradle b/metamorph/build.gradle index a14f783d3..0f57ceea3 100644 --- a/metamorph/build.gradle +++ b/metamorph/build.gradle @@ -22,6 +22,7 @@ dependencies { api project(':metamorph-api') implementation project(':metafacture-commons') implementation project(':metafacture-flowcontrol') + implementation project(':metafacture-io') implementation project(':metafacture-mangling') implementation project(':metafacture-javaintegration') implementation 'org.slf4j:slf4j-api:1.7.21' diff --git a/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java b/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java index 1972e362a..538549f1b 100644 --- a/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java +++ b/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java @@ -16,6 +16,7 @@ package org.metafacture.metamorph.maps; +import org.metafacture.io.FileOpener; import org.metafacture.metamorph.api.MorphExecutionException; import org.metafacture.metamorph.api.helpers.AbstractReadOnlyMap; @@ -24,11 +25,10 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.Reader; import java.io.UncheckedIOException; import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -53,6 +53,7 @@ */ public final class FileMap extends AbstractReadOnlyMap { + private final FileOpener fileOpener = new FileOpener(); private final Map map = new HashMap<>(); private Pattern split = Pattern.compile("\t", Pattern.LITERAL); @@ -100,6 +101,33 @@ public void setFile(final String file) { Collections.addAll(filenames, file); } + /** + * Sets the encoding used to open the resource. + * + * @param encoding new encoding + */ + public void setEncoding(final String encoding) { + fileOpener.setEncoding(encoding); + } + + /** + * Sets the compression of the file. + * + * @param compression the name of the compression + */ + public void setCompression(final String compression) { + fileOpener.setCompression(compression); + } + + /** + * Flags whether to use decompress concatenated file compression. + * + * @param decompressConcatenated true if file compression should decompress concatenated + */ + public void setDecompressConcatenated(final boolean decompressConcatenated) { + fileOpener.setDecompressConcatenated(decompressConcatenated); + } + private void loadFiles() { filenames.forEach(this::loadFile); } @@ -107,10 +135,11 @@ private void loadFiles() { private void loadFile(final String file) { try ( InputStream stream = openStream(file); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)) + Reader reader = fileOpener.open(stream); + BufferedReader br = new BufferedReader(reader) ) { String line; - while ((line = reader.readLine()) != null) { + while ((line = br.readLine()) != null) { if (line.isEmpty()) { continue; } @@ -127,10 +156,9 @@ private void loadFile(final String file) { private InputStream openStream(final String file) { return openAsFile(file) - .orElseGet(() -> openAsResource(file) - .orElseGet(() -> openAsUrl(file) - .orElseThrow(() -> new MorphExecutionException( - "File not found: " + file)))); + .orElseGet(() -> openAsResource(file) + .orElseGet(() -> openAsUrl(file) + .orElseThrow(() -> new MorphExecutionException("File not found: " + file)))); } private Optional openAsFile(final String file) { diff --git a/metamorph/src/main/resources/schemata/metamorph.xsd b/metamorph/src/main/resources/schemata/metamorph.xsd index 7d3cc7cd0..0fe398e1e 100644 --- a/metamorph/src/main/resources/schemata/metamorph.xsd +++ b/metamorph/src/main/resources/schemata/metamorph.xsd @@ -602,6 +602,21 @@ Allow empty values in Map. + + + Sets the compression of the file. + + + + + Flags whether to use decompress concatenated file compression. + + + + + Sets the encoding used to open the resource. + + Filename(s) referencing the lookup table(s). Can be one diff --git a/metamorph/src/test/java/org/metafacture/metamorph/maps/FileMapTest.java b/metamorph/src/test/java/org/metafacture/metamorph/maps/FileMapTest.java index e55b06679..ee92795bf 100644 --- a/metamorph/src/test/java/org/metafacture/metamorph/maps/FileMapTest.java +++ b/metamorph/src/test/java/org/metafacture/metamorph/maps/FileMapTest.java @@ -46,13 +46,12 @@ public final class FileMapTest { " " + "" + "" + - " " + + " " + ""; @Test public void shouldLookupValuesInFileBasedMap() { - assertMorph(receiver, String.format(MORPH, "lookup in", ""), + assertMorph(receiver, buildMorph("lookup in", ""), i -> { i.startRecord("1"); i.literal("1", "gw"); @@ -70,7 +69,7 @@ public void shouldLookupValuesInFileBasedMap() { @Test public void shouldWhitelistValuesInFileBasedMap() { - assertMorph(receiver, String.format(MORPH, "whitelist map", ""), + assertMorph(receiver, buildMorph("whitelist map", ""), i -> { i.startRecord("1"); i.literal("1", "gw"); @@ -89,7 +88,7 @@ public void shouldWhitelistValuesInFileBasedMap() { @Test public void shouldReplaceValuesUsingFileBasedMap() { - assertMorph(receiver, String.format(MORPH, "setreplace map", ""), + assertMorph(receiver, buildMorph("setreplace map", ""), i -> { i.startRecord("1"); i.literal("1", "gw-fj: 1:1"); @@ -107,7 +106,7 @@ public void shouldReplaceValuesUsingFileBasedMap() { @Test public void shouldReplaceCommaSeparatedValuesUsingFileBasedMapSetting() { - assertMorph(receiver, String.format(MORPH, "setreplace map", "separator=\",\""), + assertMorph(receiver, buildMorph("setreplace map", "separator=\",\""), i -> { i.startRecord("1"); i.literal("1", "gw"); @@ -125,7 +124,7 @@ public void shouldReplaceCommaSeparatedValuesUsingFileBasedMapSetting() { @Test public void shouldReplaceEmptyValuesUsingFileBasedMapSetting() { - assertMorph(receiver, String.format(MORPH, "setreplace map", "allowEmptyValues=\"true\""), + assertMorph(receiver, buildMorph("setreplace map", "allowEmptyValues=\"true\""), i -> { i.startRecord("1"); i.literal("1", "zz"); @@ -141,7 +140,7 @@ public void shouldReplaceEmptyValuesUsingFileBasedMapSetting() { @Test public void shouldNotReplaceEmptyValuesUsingFileBasedMapSetting() { - assertMorph(receiver, String.format(MORPH, "setreplace map", ""), + assertMorph(receiver, buildMorph("setreplace map", ""), i -> { i.startRecord("1"); i.literal("1", "zz"); @@ -154,4 +153,65 @@ public void shouldNotReplaceEmptyValuesUsingFileBasedMapSetting() { } ); } + + @Test + public void shouldLookupValuesInGzipFileMap() { + assertMorph(receiver, buildMorph("lookup in", "file-map-test.txt.gz", ""), + i -> { + i.startRecord("1"); + i.literal("1", "gw"); + i.literal("1", "fj"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("1", "Germany"); + o.get().literal("1", "Fiji"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldNotLookupValuesInBlockedGzipFileMapWithoutDecompressConcatenated() { + assertMorph(receiver, buildMorph("lookup in", "file-map-test.txt.bgzf", ""), + i -> { + i.startRecord("1"); + i.literal("1", "gw"); + i.literal("1", "fj"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().endRecord(); + } + ); + } + + @Test + public void shouldLookupValuesInBlockedGzipFileMap() { + assertMorph(receiver, buildMorph("lookup in", "file-map-test.txt.bgzf", "decompressConcatenated=\"true\""), + i -> { + i.startRecord("1"); + i.literal("1", "gw"); + i.literal("1", "fj"); + i.endRecord(); + }, + o -> { + o.get().startRecord("1"); + o.get().literal("1", "Germany"); + o.get().literal("1", "Fiji"); + o.get().endRecord(); + } + ); + } + + private String buildMorph(final String data, final String options) { + return buildMorph(data, "file-map-test.txt", options); + } + + private String buildMorph(final String data, final String map, final String options) { + return String.format(MORPH, data, map, options); + } + } diff --git a/metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.bgzf b/metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.bgzf new file mode 100644 index 0000000000000000000000000000000000000000..b49870fcdf7227a19af76ba743030fb5f27472bb GIT binary patch literal 16392 zcmZ`>cYsvY)qgAqyY7=BXvBuP3ru;1wg^a5Ec5DEKCneJBoJ^B{+de!?+oJxE(0$-V#m>^0))wXndx+%m3#V%9CP(rZe zwK#CiKVKT%hxnAqpM_jeF~Q{TJonW_l>8x(rDMA0MeL*?DR#Pq8@QtPkKG840g&cn zf*N(gmaTzj>5O#aaRebSu>qYXJGR9zncq|xO<^*?rxOx}lhk*Vs$#%MSo$t}aJcH_ zD^DO8OZY%Y#fo3p<-72A8wAc)$Nt)rAP<5(6{}H2RkdxEsdnzt4?|rEN+wC@DAHEe zZ%rIKpOOUV+7g3Ta7_q2bN}pbDRdbUK}M#lwgI}@dz*fwkYIEmYenq%2gD~RI(BRt zbe0@a!S2Ef7Wbu&?F+Iz$UKk1U-e{6(PcoW7~!vAf$bjhcz6b^yKxPkUv^dEF@Kz)nrY0JW-HG9X;}(n}J>P6nNsi?J{l z+-lbgI)<Hn3fhYa(NGG4 ztJqo)+9hIJ|9CtB>0nT&V+q+!>IjdXHz*SudJ^#s@J-}n3C|Mzs?m4f zxW7=qL6A^A?|PQ!c;B}QvgP2QPbfj)(MV1qF~L&Eq2QheD369?g>sNKC`Ek<3&HmtAoTsm z-*usMf;~}7O(>+I1b?rIRzyrR`@sB;BT|$-Q65bs9TUYjDJOiJj@&z8I8`12nVABm zk%(HX;;vq~>Hq~9tck#O*3akVIYHi7$Sd`*h`r1m|J+BH#Rl;y6)B6)e5$Q3bl! zmhNAlMtlf}-FjcQ)_VT>puZ1!bq5YxH~Fd8DTom=ZYCxrqBsL&yR@ZCDYrmPaRG0Q z4b!zo%q@q1AERmtpmBjQ;;Wc7ttDrEN_>#kG;k1D>-{JWh%ffWx(E%6RZyPJf}-1 z3U-JgrDCouDLjff^cr~#Rv38VL(s)zkvra1-x*bke!{CF}E0`P*mOsq}__(|@VTmETKUI2OP zShCRUxo9gO?}T1YlvDB|8!FJ=5?I$GS<&fjt-y-WOFK~65WE#J8&i;IqXH|p`>yCm z_UUi|D+juyAtazt#YrbEp4cC*CMG6?pvubMs|(+3qJ(5*xtJjtXsc2sTzbn-BpKQ&BOen5 z1GlGyZcoJ(1?)?>-CynLO0`w!aGWe}*^b<#h!S(G`YxB*RELAOc9OZ6gI>ti;LV3G zoc1CG00b3NF;TKSzna`#a=3!h2kb-?{Z_iw?=I{_v4emjPAxO-pjK49>+6*lwIiU| z5EHGqEyr1CuDPVoe<+R!O)V4CghcdUYlw85manBklhNj5Vb#&xYS}@T1is4oyWgc~ zp?ESAOWMgIKJQF_Z$G6tqCJ@cSpuvso^ZusD%x=@pw?`QwQ!9>=cl&*o;!W;yEW7$ z5I`o8X_Cy|EHAH_dB%y9+NgQSY{Ahb#G~lyJE}DUiBwRGGO5FYtsgg zr8Hb$#Go|pt+CnHIA?G5SEo?&AU}jtAh%JxOSm6DA9X2}A1(q_j>k1$70u%9UlC2T zY{d(UAQ++v7U!UZY#jF5bLHHExBBc6h-?Jy!woSjp`%4(_ktsUK%$_+{Q$-zl$g7WGP~l(0woa;}@(_}rmx+0LNVS8CfJ|`T(u$L)#L5Vngcv>) z9kO)R&wrr^VK5IeqOW7X%)9iqa>`&sK$VVpwm%1AW!S>C5&_g<18VR`GYcI~A*{nR z%EzL5g0&U%?!00Yr4b@ru8-l0uAm_nBXF9J{$Q=3A{>&UafV6;bFGaplntjOMFy60 z&6aNM(%94`9hkJ6%D`}N89hMJH`KA z*{~t(l1rMtq#~WEl#aDz=~tx#B>3<)894fGgahwCaL#JV+rVsnCMMSw^VImcFV<7| zK>qd6wnpVqas}LJUwZqlGb#D|aml!lEkuN zO;o@P0Wu;F3ZH zS_q&Sv*{I^@^(g(0u5`lMy@_#07V1Lgj_+8RNX?6|cc@?@bwa z83o2j5CK@ms;mwv9jzcHwt^xgYE*;urR#)J$PpH3aMao!mV&LWN)&LwMn=y+2wEKH#FgkI1@5q)`c z_dIjc{$oxjz61mrds-{mUPnv7iDF~ zfutO=K7tH4JuvD-s>cg+NjG1lEp^dxFxq8ze|HK+3nV28S~bs-S6wxCH-#h+0+l32 zZ2@M{HXoTlA;hSdkgl9WCb|3?=?01l!CD88XIDDIt%GdCM-OPAf)!ZSr3z+^YP*3V zmN$NI^(PdTaGttQGihlHloPm}FH_C~whGR-WJkbQi=~w=BQtv`wg5*-6)YXIN@nwS zVrBg(a2Q#(pcq(Eu@)i}mT#xXAf#0fZiX9BoUo)gEF6x!`Scx>kO0Zb6fD6C7{%4s zR~>#e+X2Z!x;lzd-TTf2N}Ca2iQBnT8)Ho7PR@NdumN4%pP) z09ySZn@oFO?+~bvgoz9_SAcy4E6Ua!cO0AC4jd3+SR#O;44F?!Hx*OM{%)=+kpHFP(Tqgxo!a)-kES6TvUOP2U zoeQ^eas|&|+%`BB?^ABYXc~YCyAD;KTt1zmVe&KFf~+6@(=R`dq~r%24q6UOP;iBm z8Rg)ib55khVw8ENfZ}L|oB$@vuV1%_qGXuJ;3-ImId-MK;FyU^C`?eW#Cql_p654E zpWnD^5!LVDXfXrhhR{Ia8VWb49CNS}bu_|51j=$EYBJ|rvvkfiRCqu_5eXg=c3wVv zwk~nM?z*xkRWF!?@KUC=pE~~?8}W2|5;A9o`(ZH7g0A<2;~#HN!Za8v7*~R9&I_OJ zpd^%WFyb6^3bs0J7obP0H=RJuj1Y7p+}c(I4_xEBZ>XlC1XQ(f>Dss-nvZ<iQ8Y`rb-0mIdJvb?GU%RKmLD@PIL11F#pwgJ zBhRG=ib-sP3Yvjn@z>q64lQ0vIT5lLMzsajB~)Y2;vv67=D$=Z1E?&9WJnTBU}xjf1ydOOI`nnwA4?G$~^l3TNqX){9$Kt=RI+&u2;8%|D4F zsh1XrP%U3b>QNDCXxqMP=TZ@hI?C=yn)pM+76WeFpRTT)Kb0)@aGRu^^VMUuuJ!)1 z{>TT*D7S$LlMH3?2RtP)YXRKti^|AJ9uvc48s3SOS6K3N7jSf5-ERR^&Vh2Wfzg__ zVl)pOOQ$zIL!n#++M+?2#Fz@BNom`Ye!4!YQWI|`II02!Z}H}bE8+(`u-LL&)V$f0NrP`Kxm>+YjM0~QO{eYpLe?7yvk?Wn3c%Ia{!b@c#3O?p;wmjX%LU$$+foKONWO0yu%=vmz2MCJryj2K&t2XKmud&JB%RqR;|wo${P1fqpDJWe3IiM zw%hz{t;=0+bf==AFX$Thc=KB3z}^2jk;*2F3MgP9#~frW# zy*r=_btW(k)C1Fi0WZF@Wqi5l+!Jo1Fb#q_#Rm*AWOYmU?ZBIV-hM9S?h+`W3p-!N z3g9@lm_ohkGUrf=ila)9=b;oXowyu++_4K49nfiY#<#z0uzYJje&a`!PGF{IcwO+P z>bw7S>OUxEFu87Ic)>87aISHOVrOx*oLZy zqQEEJ=iSW|9XvjP@Jz&`N^v0uyWc1EPf!K4A80{}3xnTyeiNEChKhKo9?J6mLRdM09 z8Gfzc)CtYIzFjSMb6j}7JpI}KP?;Y#HZwfC!&m!TaU8JxnB$|Aje)V5;w6l-@!}1* zoxd@47lkp@GjqIY+hM#y3|YfA4eDR@#)5M0k|pIJ!3%jtLjUj(rW^V)fW@nM55Yg_!az(i2O}_ADD%#;?ZVudBm+f#$wsyUf@-p~~ z3;1>1wF?Svm>}JMc9*Lue_`SQQ;bbffY~H(!>S&$s7lRvrg3t((a?pE!bYrTJyvxq z#S`V#)!#p@@{@(Ml>}qTIAmu@QNjdhL-Y zMF5c0;73%We_4l|pS`o?ED9hdrv-q}&lbChnL941q~v5gVI84m29M*%87tc9lCwn(f#EAUawz9Qy9T8bs3&5Ga%g5 zAMTz)xh;gN1Kysn2IJMD$`5N_yhG`MAXhTHVkcVLjD7pI45ES@sRNbKC+*rY2cB{N zz#dfUK)0jh_@J1O@B(P6Y3r_x7TRAG@0c`oICU{VC#U3jE9xOA#JW$mQk@*KtyRxk z5xd#nt{i^;NmRB4mR5!jI>;Ifre8-+^C&E#5!?b&y4aC7H(A?G=%1rBLPECS!$soO z*Yg9mUCmPonF-LqTORget|t0(og$7u_1di1Kk z#6=NNgF*$o{Yk-)>-rx{MGd1$gG$5F6HhB|`@TJ@H37g^8(|yK{dM1^DPHidHO#-) zVcEdk=E2DqUqyKrAjX3&W4#bS!T)MuMTCMF`lI*r5i4W2LGh3yn7i=vmgdK~tuK#W z-aB=8JNg@W9_7(*aCY}Qaw;YBFfQz5ipMmvqGx#r**R?wNT&!-kD*>i4#$Ufvw-xM zJNItksg)zLwDa2!f&7TOmrkOxGy=g%^NyqzC+gjs9yyK*1VHbEQ*Ct1E}7ATf_^j? z7VG~9)$;%LsZ;+@@>vDpQ^4`Ae-TH1L7ffIYBRhT!WD?!S3Y?7v}-BQ zkQ8cDe66ZFEjPs5OE!>01SnG*Dw4LQ?g`IK{E$)xISIc);gi^%7Q>+XKMs*A#K9oM zqkXmmaZ7#jHm^Ic)3aIXDKtPpr=sB@v}sRwg*DLb?}v80K!L#I$nu(_)rJg*_4q$$ zE(lS%aTFI$W{UDQ3*h>R=BU_%?Ut zj#x1Dy{T6ZQ%(=!a6Pn4*s>_a72RZmfthpe8A}mE&#(&l@%B~ri~3UQobi;NjBBn% z1pgKT5iWdV+)#=u0*a+iO-x{&fHW2U^2L{^fI?!&vp7DA-|gYYS(dlGb#%3Na`{bE zVu#m7Q>;mwir&Iv?cvG%>~2&kLY0Z4x)uAm>BUEWe$#0kiS4JV3@-oCG#|y=6@j%4 z2r7q-*I#}4^z*38KP2%WyC;~g7j~H>y@&4HLrHQrD4S)rlfHKx^j{>DCj`7zE#cmN zd-6VNX3kDPLu#{kZpAjLw{v3!bpq;;hh8FT`$h4<;+5x6*&2yLX1759=VFgvGk{DK zu(OO>57^4ug7C`)pEtE*=k>r~z3n-+hLmz^kJl$rc0Pw&7r|rr047?NAr!0c$4C7F ziCxW)fAO|o&mgkq5PXXl@=;gbb-*+n%hq{d^;bN%rVTi_}dAcp&|$TpEb z7Lh>L+{U{?FnCB4@%jgC_wd`dP&Q}eZ1)v%BevBx%zI;6^|KVY0CFtA^kB56O=SC? zX@@8*hk-Z>kYv>^GmU2Yu1Eb6bya20VN@-x;MT=4DfX9z^>dUbsMd8TXfiN8E_AJW z&S?8RX>r8Rh{qqSf%$I!cU6?LR)MmJ zd0CuYT5)A>bg)!&ELIV_u2LsbQxUjOoLyb*a2b9ghPMugh@cg@!+!${r(jkV)1%*{ zTquBVu`$=pmIcjNj)VE86i@|gP0dziA*SZ0RQ)wWdrs(uTcH|ATU69vDq(7+~{CO u)AIIV*v_ie)PkTdw@%H>V;ga1S$9H5@;P)LZr$1x{=MpqHw^Y40Q^4}^(27+ literal 0 HcmV?d00001 diff --git a/metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.gz b/metamorph/src/test/resources/org/metafacture/metamorph/maps/file-map-test.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..ea631ec4e4d27a44b4b7cc48e41f52e6a62a5f57 GIT binary patch literal 3088 zcmV+r4Da(FiwFpaP@iD{17>M#Wi4%Ca4mFYb963rcys`LSj%qQIuc!zKQayw1Q=j8 zcXqbpB#!OG*X<;@*-;c*Vv7%%4?X&a26sPykRO{rDy4&P^qp z7F<-4U=43c_gNE=8}CFpnw(oqU+NP>|B7Koa-t zeBZQ?9+PlfH0(Sy$|dcvr`fq{vNmRcgH=%C>_PRpGFrX@GHh74d$W7<@P>;6J3p#W z34I`Ohwx&#k#rfyi`YS`DPScmVEf`U2e|ej0Q7|RgnR;&bdv?{cp*ZSYajz8gig39 zaP7#>Ev_YTmarbX>Ddfi}wR*Pm##`@B zOo0@It*Z#C6h}cEiCwgroL^;Zn?{g@ssR*vP-)@i)ksRi+K@6r%&d&ZB0hV*nxJiV zN^=As;!#%>Zu3c{R)ytw2@hPt6?-2jm}P~Pt$aq@Mgot{@WwT=6D(hts@ZfV~%65@=3f6!A{kMPo2E1mM!Gji34>*3d;$$7! zK_Ki!E|rG(jpEXhk}7!bxO9M>K-5XkE^`n+{v|x%T=)XwgUg;s5(3E8SFT3n=P;iR zcmg337GWdTs7?KDGNll7Uq4{j{Q=-X^L+;v(-H2(g@W`BzSMB?-m*I_15AY|KYOR(O_5Z+OZXdK@qND@ zaPQ4rX!t$=C7e`z9~tGbief20Dk@~yQfQvCihGbDdURAcGqJY^UHP0eZ;&GIl@AT1 zH5I$jNFzb`n!U$i6nl*V1(Mj!XICPoSG;N2`twOUgQ!BBH$8g~pCQ9@uXq!RAY7`6n{F?9Koi4;-6}zoh(9SSP_>&( z3(K2J49p8Tb{_Z~6kP*Xi`H|k*mYQunw472H6q~} z_HkZEu3cWx&{TT@m#Ei2<2thI7=$uhCw7gdi=*?3+Rcc36f14ejnQRP zO=$?>Na`Q|Mr8f%dhi-ar^l7!qo>RM70X_EGfE=AhX} z-#W^j^W=@;2a6pmD02@Ept}>Z0-a~eQM^;^M!8w=u3i8$vvg&&`7hvg~Hw9YJ~&UAvrh z4C`_|%D{KGKI>S_Zz^`Ht1`P7#cmY_BZRDJfkg>T3b$1#Qbq%j+FZsa1NXdGxvfyb z;YD2)+$}x0e26V6={S&{S|WtbAcoPBj7YJnhAZ>w%-E zEzDfFm3<-=F%DAM;x=F>cuC5~g@bH>?>O~qj!Cs6MG!J4Jh}>az`J_<$XsUE*Y$Yr)h2RoU z2$f;MycHc?-l>;TxZ6vt$Gyi4M;QFviYk;I;~Mr!V^-#_r6a}Az4s!Ka__9Wqs#x8f7@0=}yR(m+dz6yt$nz{4D zG73oVLD37t-4j$qcnFNf+=WG7k{)q9%)XPJ1|FB>a@vS)yG@yzjEAG(F38TH$W9Cx zIov=B4AgPl_&>FrXlL)uI)!jI7q$QJp*t&$@)dVS!hBaH-MEJpKci)HkFxL?y|?w4 zN4=BmGqp$Bz(oH!V4ii{CxpT56J~=RNp-KBi!*vZnXZ@YUMGns$U4CBUV#F)==ba% z#z2+d)a;wJ_go-a1t?l}kLix$ma1uwrf?rL`u0_gsi)rLb&%67@1ou?wE2Q{z&d)c z^d7j7aswCiJ+OU786MXz*9Uf=(PsSvIj~PFEqcEyBMOsJQ_e$ zL~j+f(P(i>iFoIEbi|wP1TKM&868~;#U??gk)(L&Ys)(`_H;OUP<&#$k=}1)d5lRr zlte@hkCA<&fj8il+QRcZro}=_rNt5|A!ipp7~;9j^Ei;q)0-uZSi*f^LMjTY3(d^R ztij_EyE+}0I>(b>kK&-$kVcd(%e4-pIy~mk#qu=)F¬Qo|m7<9TY?V=I>1Bn@Im zEtx}E;-jwsi`*`3h)uGAfKE}YCrdiSB?1!)&!eJ-6qVx1g9_f^3H&e=MsO;`kLa0n zcw}GDKvhk4xU!l?aj2F+nxCuL?Dq)Adm;5+%ai`O=!1Hk20#c)=rPgJ6Q+}NghK9E z6v}wGG;^OgvTZqS8TQfVx4+^FUt#I)vgo5k1c{`x0`gKKaPL z94oa3NX8&mma7{Fl;E!z0yUXxsJ2fR3u;qhkmywOF~wc#H1k2RugJ+`hWGiP*-ym9 zSwHDGSWu`$Niw(%d|h&w4?QHKLFCvQ6n78@0TNjrlzfOM@=`bWfa~I&Y%!AX5t@G! z=-RHqD7-bMu_|eJf$TuijVQQlRz80+8u#GCBY=|P9Sn#%HKL+^gY~_O#C2F?2-lU{ zEMxS89xyPB9hkeV@{Wdmzs3YH3w~G1j#wvkgT7Rn()&s08Zk>^kR1_uA7ZLWKZ> ze-+M#qt?$!)AVZ Date: Tue, 28 Jun 2022 10:10:56 +0200 Subject: [PATCH 2/2] Update documentation for `FileMap`s new encoding option. (#455) --- .../java/org/metafacture/metamorph/maps/FileMap.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java b/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java index 538549f1b..53aa28c03 100644 --- a/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java +++ b/metamorph/src/main/java/org/metafacture/metamorph/maps/FileMap.java @@ -38,8 +38,11 @@ import java.util.regex.Pattern; /** - * Provides a {@link Map} based on files. Can be one file or a comma separated list of files. - * The files are supposed to be UTF-8 encoded. The default separator is {@code \t}. + * Provides a {@link Map} based on files. Can be a single file or a + * comma-separated list of files. + * + * The default {@link #setEncoding encoding} is UTF-8. + * The default {@link #setSeparator separator} is {@code \t}. * * By setting {@link #allowEmptyValues} to {@code true} the values in the * {@link Map} can be empty thus enabling e.g. @@ -194,7 +197,7 @@ private Optional openAsUrl(final String file) { /** * Sets the separator. * - * Default value: {@code \t} + * Default value: {@code \t} * * @param delimiter the separator */