diff --git a/CHANGELOG.md b/CHANGELOG.md index 0729697ea..4b3875666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Removing deprecated classes and methods [#540](https://github.com/ie3-institute/PowerSystemDataModel/issues/540) - Refactor CSV data sources [#716](https://github.com/ie3-institute/PowerSystemDataModel/issues/716) +- Deleted parameter initFiles, set parameter append to false by default [#791](https://github.com/ie3-institute/PowerSystemDataModel/issues/791) ## [3.0.0] - 2023-02-16 diff --git a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java index 9ad2cacf7..e635a5f7d 100644 --- a/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java +++ b/src/main/java/edu/ie3/datamodel/io/connectors/CsvFileConnector.java @@ -117,19 +117,11 @@ private BufferedCsvWriter initWriter(String baseDirectory, CsvFileDefinition fil if (!directories.exists() && !directories.mkdirs()) throw new IOException("Unable to create directory tree '" + directories + "'"); - File pathFile = new File(fullPath); - boolean append = pathFile.exists(); BufferedCsvWriter writer = new BufferedCsvWriter( - fullPath, fileDefinition.headLineElements(), fileDefinition.csvSep(), append); - if (!append) { - writer.writeFileHeader(); - } else { - log.warn( - "File '{}' already exist. Will append new content WITHOUT new header! Full path: {}", - fileDefinition.fileName(), - pathFile.getAbsolutePath()); - } + fullPath, fileDefinition.headLineElements(), fileDefinition.csvSep(), false); + writer.writeFileHeader(); + return writer; } @@ -220,7 +212,7 @@ public BufferedReader initReader(String filePath) throws FileNotFoundException { * possible readers will be initialized. * @return A mapping from column scheme to the individual time series meta information */ - public Map + public Map getCsvIndividualTimeSeriesMetaInformation(final ColumnScheme... columnSchemes) { return getIndividualTimeSeriesFilePaths().parallelStream() .map( @@ -228,7 +220,7 @@ public BufferedReader initReader(String filePath) throws FileNotFoundException { /* Extract meta information from file path and enhance it with the file path itself */ IndividualTimeSeriesMetaInformation metaInformation = fileNamingStrategy.individualTimeSeriesMetaInformation(filePath); - return new edu.ie3.datamodel.io.csv.CsvIndividualTimeSeriesMetaInformation( + return new CsvIndividualTimeSeriesMetaInformation( metaInformation, FileNamingStrategy.removeFileNameEnding(filePath)); }) .filter( diff --git a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java index ec50a044f..45dc40ccc 100644 --- a/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java +++ b/src/main/java/edu/ie3/datamodel/io/sink/CsvFileSink.java @@ -59,7 +59,7 @@ public class CsvFileSink implements InputDataSink, OutputDataSink { private final String csvSep; public CsvFileSink(String baseFolderPath) { - this(baseFolderPath, new FileNamingStrategy(), false, ","); + this(baseFolderPath, new FileNamingStrategy(), ","); } /** @@ -69,17 +69,10 @@ public CsvFileSink(String baseFolderPath) { * * @param baseFolderPath the base folder path where the files should be put into * @param fileNamingStrategy the data sink file naming strategy that should be used - * @param initFiles true if the files should be created during initialization (might create files, - * that only consist of a headline, because no data will be written into them), false - * otherwise * @param csvSep the csv file separator that should be use */ - public CsvFileSink( - String baseFolderPath, - FileNamingStrategy fileNamingStrategy, - boolean initFiles, - String csvSep) { - this(baseFolderPath, new ProcessorProvider(), fileNamingStrategy, initFiles, csvSep); + public CsvFileSink(String baseFolderPath, FileNamingStrategy fileNamingStrategy, String csvSep) { + this(baseFolderPath, new ProcessorProvider(), fileNamingStrategy, csvSep); } /** @@ -94,22 +87,16 @@ public CsvFileSink( * @param baseFolderPath the base folder path where the files should be put into * @param processorProvider the processor provided that should be used for entity serialization * @param fileNamingStrategy the data sink file naming strategy that should be used - * @param initFiles true if the files should be created during initialization (might create files, - * that only consist of a headline, because no data will be written into them), false - * otherwise * @param csvSep the csv file separator that should be use */ public CsvFileSink( String baseFolderPath, ProcessorProvider processorProvider, FileNamingStrategy fileNamingStrategy, - boolean initFiles, String csvSep) { this.csvSep = csvSep; this.processorProvider = processorProvider; this.connector = new CsvFileConnector(baseFolderPath, fileNamingStrategy); - - if (initFiles) initFiles(processorProvider, connector); } @Override @@ -352,40 +339,6 @@ private void write(C entity) { } } - /** - * Initialize files, hence create a file for each expected class that will be processed in the - * future. Please note, that files for time series can only be create on presence of a concrete - * time series, as their file name depends on the individual uuid of the time series. - * - * @param processorProvider the processor provider all files that will be processed is derived - * from - * @param connector the connector to the files - */ - private void initFiles( - final ProcessorProvider processorProvider, final CsvFileConnector connector) { - processorProvider - .getRegisteredClasses() - .forEach( - clz -> { - try { - String[] headerElements = - csvHeaderElements(processorProvider.getHeaderElements(clz)); - - connector.getOrInitWriter(clz, headerElements, csvSep); - } catch (ProcessorProviderException e) { - log.error( - "Error during receiving of head line elements. Cannot prepare writer for class {}.", - clz, - e); - } catch (ConnectorException e) { - log.error( - "Error during instantiation files. Cannot get or init writer for class {}.", - clz, - e); - } - }); - } - /** * Transforms a provided array of strings to valid csv formatted strings (according to csv * specification RFC 4180) diff --git a/src/test/groovy/edu/ie3/datamodel/io/csv/GridIoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/csv/GridIoIT.groovy index 57fc582ee..9b66b4578 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/csv/GridIoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/csv/GridIoIT.groovy @@ -40,7 +40,7 @@ class GridIoIT extends Specification implements CsvTestDataMeta { new DefaultDirectoryHierarchy("output", "vn_simona")) tempDirectory = Files.createTempDirectory("GridIoIT") sinkFlat = new CsvFileSink(tempDirectory.toAbsolutePath().toString()) - sinkHierarchic = new CsvFileSink(tempDirectory.toAbsolutePath().toString(), hierarchicNamingStrategy, false, ",") + sinkHierarchic = new CsvFileSink(tempDirectory.toAbsolutePath().toString(), hierarchicNamingStrategy, ",") } def cleanupSpec() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy index ccd738a5e..f56a5a92c 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/sink/CsvFileSinkTest.groovy @@ -72,33 +72,6 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { } } - def "A valid CsvFileSink called by simple constructor should not initialize files by default and consist of several default values"() { - given: - CsvFileSink csvFileSink = new CsvFileSink(testBaseFolderPath) - csvFileSink.shutdown() - - expect: - !new File(testBaseFolderPath).exists() - csvFileSink.csvSep == "," - } - - def "A valid CsvFileSink with 'initFiles' enabled should create files as expected"() { - given: - CsvFileSink csvFileSink = new CsvFileSink(testBaseFolderPath, - new ProcessorProvider([ - new ResultEntityProcessor(PvResult), - new ResultEntityProcessor(EvResult) - ], [] as Map), - new FileNamingStrategy(), - true, - ",") - csvFileSink.shutdown() - - expect: - new File(testBaseFolderPath).exists() - new File(testBaseFolderPath + File.separator + "ev_res.csv").exists() - new File(testBaseFolderPath + File.separator + "pv_res.csv").exists() - } def "A valid CsvFileSink is able to convert an entity data map correctly to RFC 4180 compliant strings"() { given: @@ -140,7 +113,7 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { csvFileSink.shutdown() } - def "A valid CsvFileSink without 'initFiles' should only persist provided elements correctly but not init all files"() { + def "A valid CsvFileSink should persist provided elements correctly"() { given: CsvFileSink csvFileSink = new CsvFileSink(testBaseFolderPath, new ProcessorProvider([ @@ -166,7 +139,6 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { new InputEntityProcessor(EmInput) ], [] as Map), new FileNamingStrategy(), - false, ",") UUID uuid = UUID.fromString("22bea5fc-2cb2-4c61-beb9-b476e0107f52") @@ -237,7 +209,6 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { CsvFileSink csvFileSink = new CsvFileSink(testBaseFolderPath, new ProcessorProvider([], timeSeriesProcessorMap), new FileNamingStrategy(), - false, ",") when: @@ -311,7 +282,6 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { ProcessorProvider.allEntityProcessors(), new HashMap, Value>, TimeSeriesEntry, Value>>()), new FileNamingStrategy(), - false, ",") when: @@ -331,7 +301,6 @@ class CsvFileSinkTest extends Specification implements TimeSeriesTestData { testBaseFolderPath, new ProcessorProvider(), new FileNamingStrategy(), - false, ",") when: