From 697f9da78caae3e68182cfcddeb129ed83ce9a8c Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 17:38:26 +0200 Subject: [PATCH 01/20] Alter scheme for COSMO weather in test files --- .gitattributes | 2 +- ...sdmTimeBasedWeatherValueFactoryTest.groovy | 28 +++++++-------- .../CouchbaseWeatherSourcePsdmIT.groovy | 4 +-- .../InfluxDbWeatherSourcePsdmIT.groovy | 2 +- .../couchbase/{ => cosmo}/weather.json | 36 +++++++++---------- .../influxDb/cosmo/weather.txt | 13 +++++++ .../testcontainersFiles/influxDb/weather.txt | 13 ------- .../sql/{psdm => cosmo}/weather.sql | 18 +++++----- 8 files changed, 58 insertions(+), 58 deletions(-) rename src/test/resources/testcontainersFiles/couchbase/{ => cosmo}/weather.json (55%) create mode 100644 src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt delete mode 100644 src/test/resources/testcontainersFiles/influxDb/weather.txt rename src/test/resources/testcontainersFiles/sql/{psdm => cosmo}/weather.sql (66%) diff --git a/.gitattributes b/.gitattributes index c0b77356d..6ef183f09 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,3 @@ # The following file will be copied to a unix Docker image and imported to InfluxDB data base. Therefore, the line # ending plays a crucial role. This prevents the endings from being adjusted with 'core.autocrlf=true' -src/test/resources/testContainerFiles/influxDb/weather.txt eol=lf +src/test/resources/testContainerFiles/influxDb/cosmo/weather.txt eol=lf diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy index b28d89f33..d89821a95 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy @@ -22,13 +22,13 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "time" : TimeUtil.withDefaults.toString(time), - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "time" : TimeUtil.withDefaults.toString(time), + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) @@ -55,13 +55,13 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ - "time" : TimeUtil.withDefaults.toString(time), - "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", - "diffuseirradiation": "282.671997070312", - "directirradiation" : "286.872985839844", - "temperature" : "278.019012451172", - "winddirection" : "0", - "windvelocity" : "1.66103506088257" + "time" : TimeUtil.withDefaults.toString(time), + "uuid" : "980f7714-8def-479f-baae-4deed6c8d6d1", + "diffuseirradiance": "282.671997070312", + "directirradiance" : "286.872985839844", + "temperature" : "278.019012451172", + "winddirection" : "0", + "windvelocity" : "1.66103506088257" ] def data = new TimeBasedWeatherValueData(parameter, coordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy index 6090a738a..ec25dea07 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy @@ -37,11 +37,11 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate" + static String coordinateIdColumnName = "coordinate_id" def setupSpec() { // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/weather.json") + MountableFile couchbaseWeatherJsonsFile = MountableFile.forClasspathResource("/testcontainersFiles/couchbase/cosmo/weather.json") couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather.json") // create an index for the document keys diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy index 9f409d06a..6d2fb76e9 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy @@ -35,7 +35,7 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def setupSpec() { // Copy import file into docker and then import it via influx CLI // more information on file format and usage here: https://docs.influxdata.com/influxdb/v1.7/tools/shell/#import-data-from-a-file-with-import - MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/weather.txt") + MountableFile influxWeatherImportFile = MountableFile.forClasspathResource("/testcontainersFiles/influxDb/cosmo/weather.txt") influxDbContainer.copyFileToContainer(influxWeatherImportFile, "/home/weather.txt") def execResult = influxDbContainer.execInContainer("influx", "-import", "-path=/home/weather.txt", "-precision=ms") println "Command \"influx -import -path=/home/weather.txt -precision=ms\" returned:" diff --git a/src/test/resources/testcontainersFiles/couchbase/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json similarity index 55% rename from src/test/resources/testcontainersFiles/couchbase/weather.json rename to src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index e075cecab..e7a49e4b2 100644 --- a/src/test/resources/testcontainersFiles/couchbase/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,47 +1,47 @@ -[{"coordinate":193186, +[{"coordinate_id":193186, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":286.872985839844, - "directirradiation":282.671997070312, + "diffuseirradiance":286.872985839844, + "directirradiance":282.671997070312, "temperature":278.019012451172, "winddirection":0, "windvelocity":1.66103506088257}, - {"coordinate":193187, + {"coordinate_id":193187, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":287.872985839844, - "directirradiation":283.671997070312, + "diffuseirradiance":287.872985839844, + "directirradiance":283.671997070312, "temperature":279.019012451172, "winddirection":0, "windvelocity":1.76103506088257}, - {"coordinate":193188, + {"coordinate_id":193188, "time":"2020-04-28T15:00:00+00:00", - "diffuseirradiation":288.872985839844, - "directirradiation":284.671997070312, + "diffuseirradiance":288.872985839844, + "directirradiance":284.671997070312, "temperature":280.019012451172, "winddirection":0, "windvelocity":1.86103506088257}, - {"coordinate":193186, + {"coordinate_id":193186, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":286.872, - "directirradiation":282.672, + "diffuseirradiance":286.872, + "directirradiance":282.672, "temperature":278.012, "winddirection":0, "windvelocity":1.662}, - {"coordinate":193187, + {"coordinate_id":193187, "time":"2020-04-28T16:00:00+00:00", - "diffuseirradiation":287.872, - "directirradiation":283.672, + "diffuseirradiance":287.872, + "directirradiance":283.672, "temperature":279.012, "winddirection":0, "windvelocity":1.762}, - {"coordinate":193186, + {"coordinate_id":193186, "time":"2020-04-28T17:00:00+00:00", - "diffuseirradiation":286.873, - "directirradiation":282.673, + "diffuseirradiance":286.873, + "directirradiance":282.673, "temperature":278.013, "winddirection":0, "windvelocity":1.663}] diff --git a/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt new file mode 100644 index 000000000..982e7ea5b --- /dev/null +++ b/src/test/resources/testcontainersFiles/influxDb/cosmo/weather.txt @@ -0,0 +1,13 @@ +# DDL +CREATE DATABASE test_weather + +# DML +# CONTEXT-DATABASE: test_weather + + +weather,coordinate_id=193186 diffuse_irradiance=286.872985839844,direct_irradiance=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 +weather,coordinate_id=193187 diffuse_irradiance=287.872985839844,direct_irradiance=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 +weather,coordinate_id=193188 diffuse_irradiance=288.872985839844,direct_irradiance=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 +weather,coordinate_id=193186 diffuse_irradiance=286.872,direct_irradiance=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 +weather,coordinate_id=193187 diffuse_irradiance=287.872,direct_irradiance=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 +weather,coordinate_id=193186 diffuse_irradiance=286.873,direct_irradiance=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/influxDb/weather.txt b/src/test/resources/testcontainersFiles/influxDb/weather.txt deleted file mode 100644 index 8ee925e99..000000000 --- a/src/test/resources/testcontainersFiles/influxDb/weather.txt +++ /dev/null @@ -1,13 +0,0 @@ -# DDL -CREATE DATABASE test_weather - -# DML -# CONTEXT-DATABASE: test_weather - - -weather,coordinate=193186 diffuse_irradiation=286.872985839844,direct_irradiation=282.671997070312,temperature=278.019012451172,wind_direction=0,wind_velocity=1.66103506088257 1588086000000 -weather,coordinate=193187 diffuse_irradiation=287.872985839844,direct_irradiation=283.671997070312,temperature=279.019012451172,wind_direction=0,wind_velocity=1.76103506088257 1588086000000 -weather,coordinate=193188 diffuse_irradiation=288.872985839844,direct_irradiation=284.671997070312,temperature=280.019012451172,wind_direction=0,wind_velocity=1.86103506088257 1588086000000 -weather,coordinate=193186 diffuse_irradiation=286.872,direct_irradiation=282.672,temperature=278.012,wind_direction=0,wind_velocity=1.662 1588089600000 -weather,coordinate=193187 diffuse_irradiation=287.872,direct_irradiation=283.672,temperature=279.012,wind_direction=0,wind_velocity=1.762 1588089600000 -weather,coordinate=193186 diffuse_irradiation=286.873,direct_irradiation=282.673,temperature=278.013,wind_direction=0,wind_velocity=1.663 1588093200000 diff --git a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql similarity index 66% rename from src/test/resources/testcontainersFiles/sql/psdm/weather.sql rename to src/test/resources/testcontainersFiles/sql/cosmo/weather.sql index c5361bbf7..055698ba1 100644 --- a/src/test/resources/testcontainersFiles/sql/psdm/weather.sql +++ b/src/test/resources/testcontainersFiles/sql/cosmo/weather.sql @@ -1,33 +1,33 @@ CREATE TABLE public.weather ( time timestamp with time zone, - coordinate integer, - diffuse_irradiation double precision, - direct_irradiation double precision, + coordinate_id integer, + diffuse_irradiance double precision, + direct_irradiance double precision, wind_direction double precision, wind_velocity double precision, temperature double precision, tid serial, CONSTRAINT weather_pkey PRIMARY KEY (tid), - CONSTRAINT "weather_datum_coordinate_unique" UNIQUE (time, coordinate) + CONSTRAINT "weather_datum_coordinate_id_unique" UNIQUE (time, coordinate_id) ) WITH ( OIDS = FALSE ) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_idx +CREATE INDEX weather_coordinate_id_idx ON public.weather USING btree - (coordinate ASC NULLS LAST) + (coordinate_id ASC NULLS LAST) TABLESPACE pg_default; -CREATE INDEX weather_coordinate_time_idx +CREATE INDEX weather_coordinate_id_time_idx ON public.weather USING btree - (coordinate ASC NULLS LAST, time ASC NULLS LAST) + (coordinate_id ASC NULLS LAST, time ASC NULLS LAST) TABLESPACE pg_default; INSERT INTO - public.weather (time, coordinate, diffuse_irradiation, direct_irradiation, wind_direction, wind_velocity, temperature) + public.weather (time, coordinate_id, diffuse_irradiance, direct_irradiance, wind_direction, wind_velocity, temperature) VALUES ('2020-04-28 15:00:00+0', 193186, 286.872985839844, 282.671997070312, 0, 1.66103506088257, 278.019012451172), ('2020-04-28 15:00:00+0', 193187, 287.872985839844, 283.671997070312, 0, 1.76103506088257, 279.019012451172), From fa0fc4a8a271c95b878b3848fa49b8141334f572 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:22:05 +0200 Subject: [PATCH 02/20] Renaming of "weather data scheme" and equivalent classes and methods --- ...=> CosmoTimeBasedWeatherValueFactory.java} | 16 +++-- .../influxdb/InfluxDbWeatherSource.java | 2 +- ...moTimeBasedWeatherValueFactoryTest.groovy} | 12 ++-- ...conTimeBasedWeatherValueFactoryTest.groovy | 4 +- ...y => CouchbaseWeatherSourceCosmoIT.groovy} | 50 +++++++-------- ...roovy => CsvWeatherSourceCosmoTest.groovy} | 58 ++++++++--------- ...vy => InfluxDbWeatherSourceCosmoIT.groovy} | 62 +++++++++---------- ....groovy => SqlWeatherSourceCosmoIT.groovy} | 50 +++++++-------- ...ata.groovy => CosmoWeatherTestData.groovy} | 2 +- 9 files changed, 127 insertions(+), 129 deletions(-) rename src/main/java/edu/ie3/datamodel/io/factory/timeseries/{PsdmTimeBasedWeatherValueFactory.java => CosmoTimeBasedWeatherValueFactory.java} (88%) rename src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/{PsdmTimeBasedWeatherValueFactoryTest.groovy => CosmoTimeBasedWeatherValueFactoryTest.groovy} (88%) rename src/test/groovy/edu/ie3/datamodel/io/source/couchbase/{CouchbaseWeatherSourcePsdmIT.groovy => CouchbaseWeatherSourceCosmoIT.groovy} (59%) rename src/test/groovy/edu/ie3/datamodel/io/source/csv/{CsvWeatherSourcePsdmTest.groovy => CsvWeatherSourceCosmoTest.groovy} (71%) rename src/test/groovy/edu/ie3/datamodel/io/source/influxdb/{InfluxDbWeatherSourcePsdmIT.groovy => InfluxDbWeatherSourceCosmoIT.groovy} (57%) rename src/test/groovy/edu/ie3/datamodel/io/source/sql/{SqlWeatherSourcePsdmIT.groovy => SqlWeatherSourceCosmoIT.groovy} (51%) rename src/test/groovy/edu/ie3/test/common/{PsdmWeatherTestData.groovy => CosmoWeatherTestData.groovy} (98%) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java similarity index 88% rename from src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java rename to src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 34b571550..9314ab3e7 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -5,8 +5,6 @@ */ package edu.ie3.datamodel.io.factory.timeseries; -import static edu.ie3.datamodel.io.factory.timeseries.TimeBasedSimpleValueFactory.*; - import edu.ie3.datamodel.models.StandardUnits; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; @@ -27,23 +25,23 @@ * Factory implementation of {@link TimeBasedWeatherValueFactory}, that is able to handle field to * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ -public class PsdmTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinate"; - private static final String DIFFUSE_IRRADIANCE = "diffuseirradiation"; - private static final String DIRECT_IRRADIANCE = "directirradiation"; +public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { + private static final String COORDINATE = "coordinateid"; + private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; + private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; private static final String WIND_DIRECTION = "winddirection"; private static final String WIND_VELOCITY = "windvelocity"; - public PsdmTimeBasedWeatherValueFactory(TimeUtil timeUtil) { + public CosmoTimeBasedWeatherValueFactory(TimeUtil timeUtil) { super(timeUtil); } - public PsdmTimeBasedWeatherValueFactory(String timePattern) { + public CosmoTimeBasedWeatherValueFactory(String timePattern) { super(timePattern); } - public PsdmTimeBasedWeatherValueFactory() { + public CosmoTimeBasedWeatherValueFactory() { super(); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index a424cf894..62636117f 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -205,7 +205,7 @@ private String createTimeConstraint(ZonedDateTime date) { } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate='" + coordinateId + "'"; + return "coordinate_id='" + coordinateId + "'"; } /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy similarity index 88% rename from src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy index d89821a95..df296966b 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/PsdmTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactoryTest.groovy @@ -8,17 +8,17 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class PsdmTimeBasedWeatherValueFactoryTest extends Specification { +class CosmoTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series with missing values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ @@ -50,8 +50,8 @@ class PsdmTimeBasedWeatherValueFactoryTest extends Specification { def "A PsdmTimeBasedWeatherValueFactory should be able to create time series values"() { given: - def factory = new PsdmTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") - def coordinate = PsdmWeatherTestData.COORDINATE_193186 + def factory = new CosmoTimeBasedWeatherValueFactory("yyyy-MM-dd HH:mm:ss") + def coordinate = CosmoWeatherTestData.COORDINATE_193186 def time = TimeUtil.withDefaults.toZonedDateTime("2019-01-01 00:00:00") Map parameter = [ diff --git a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy index 93bc7b37d..fb37a49bd 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactoryTest.groovy @@ -6,7 +6,7 @@ package edu.ie3.datamodel.io.factory.timeseries import edu.ie3.datamodel.models.StandardUnits -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.util.TimeUtil import edu.ie3.util.quantities.PowerSystemUnits import edu.ie3.util.quantities.QuantityUtil @@ -73,7 +73,7 @@ class IconTimeBasedWeatherValueFactoryTest extends Specification { def "A time based weather value factory for ICON column scheme builds a single time based value correctly"() { given: def factory = new IconTimeBasedWeatherValueFactory() - def coordinate = PsdmWeatherTestData.COORDINATE_67775 + def coordinate = CosmoWeatherTestData.COORDINATE_67775 def parameter = [ "datum" : "2019-08-01 01:00:00", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy similarity index 59% rename from src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index ec25dea07..bff37efbf 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval @@ -25,7 +25,7 @@ import spock.lang.Specification import java.time.ZoneId @Testcontainers -class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @@ -62,8 +62,8 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc "--dataset", "file:///home/weather.json") def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, PsdmWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) } def "The test container can establish a valid connection"() { @@ -75,10 +75,10 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present @@ -88,49 +88,49 @@ class CouchbaseWeatherSourcePsdmIT extends Specification implements WeatherSourc def "A CouchbaseWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy similarity index 71% rename from src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 07ceff508..aad1e856a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourcePsdmTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -12,7 +12,7 @@ import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.value.SolarIrradianceValue import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -20,7 +20,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.TemperatureValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.datamodel.models.value.WindValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.geo.GeoUtils @@ -31,7 +31,7 @@ import spock.lang.Shared import spock.lang.Specification import tech.units.indriya.quantity.Quantities -class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { +class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { @Shared CsvWeatherSource source @@ -40,16 +40,16 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, IdCoordinateSource coordinateSource def setupSpec() { - coordinateSource = PsdmWeatherTestData.coordinateSource - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + coordinateSource = CosmoWeatherTestData.coordinateSource + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -58,49 +58,49 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def "A CsvWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A CsvWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } def "The CsvWeatherSource is able to build a single WeatherValue from field to value mapping"() { @@ -108,7 +108,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -152,7 +152,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -177,7 +177,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, def defaultCoordinate = GeoUtils.DEFAULT_GEOMETRY_FACTORY.createPoint(new Coordinate(7.4116482, 51.4843281)) def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", @@ -200,7 +200,7 @@ class CsvWeatherSourcePsdmTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> Optional.empty() - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy similarity index 57% rename from src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 6d2fb76e9..15f014f69 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.influxdb import edu.ie3.datamodel.io.connectors.InfluxDbConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.geo.GeoUtils import edu.ie3.util.interval.ClosedInterval @@ -22,7 +22,7 @@ import spock.lang.Shared import spock.lang.Specification @Testcontainers -class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") @@ -43,8 +43,8 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource if(!execResult.stdout.isEmpty()) println execResult.getStdout() def connector = new InfluxDbConnector(influxDbContainer.url,"test_weather", "test_scenario") - def weatherFactory = new PsdmTimeBasedWeatherValueFactory() - source = new InfluxDbWeatherSource(connector, PsdmWeatherTestData.coordinateSource, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory() + source = new InfluxDbWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } @@ -57,9 +57,9 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue) @@ -68,61 +68,61 @@ class InfluxDbWeatherSourcePsdmIT extends Specification implements WeatherSource def "An InfluxDbWeatherSource can read multiple time series values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H , PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H , CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H , PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H , CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeseries_193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeseries_193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeseries_193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeseries_193187) } def "An InfluxDbWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeseries_193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeseries_193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) } def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() { - def validCoordinate = PsdmWeatherTestData.COORDINATE_193186 + def validCoordinate = CosmoWeatherTestData.COORDINATE_193186 def invalidCoordinate = GeoUtils.xyToPoint(48d, 7d) - def time = PsdmWeatherTestData.TIME_15H - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H , PsdmWeatherTestData.TIME_17H) + def time = CosmoWeatherTestData.TIME_15H + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H , CosmoWeatherTestData.TIME_17H) def emptyTimeSeries = new IndividualTimeSeries(UUID.randomUUID(), Collections.emptySet()) def timeseries_193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H ,PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H ,PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H ,PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H ,CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H ,CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H ,CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) when: def coordinateAtDate = source.getWeather(time, invalidCoordinate) def coordinateInInterval = source.getWeather(timeInterval, invalidCoordinate) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy similarity index 51% rename from src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy rename to src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 11562eef7..d63476667 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourcePsdmIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -6,11 +6,11 @@ package edu.ie3.datamodel.io.source.sql import edu.ie3.datamodel.io.connectors.SqlConnector -import edu.ie3.datamodel.io.factory.timeseries.PsdmTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue -import edu.ie3.test.common.PsdmWeatherTestData +import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval @@ -22,7 +22,7 @@ import spock.lang.Shared import spock.lang.Specification @Testcontainers -class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestHelper { +class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer("postgres:11.9") @@ -41,15 +41,15 @@ class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestH postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) - def weatherFactory = new PsdmTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, PsdmWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { given: - def expectedTimeBasedValue = new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H) + def expectedTimeBasedValue = new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H) when: - def optTimeBasedValue = source.getWeather(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.COORDINATE_193186) + def optTimeBasedValue = source.getWeather(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.COORDINATE_193186) then: optTimeBasedValue.present equalsIgnoreUUID(optTimeBasedValue.get(), expectedTimeBasedValue ) @@ -58,49 +58,49 @@ class SqlWeatherSourcePsdmIT extends Specification implements WeatherSourceTestH def "A NativeSqlWeatherSource can read multiple timeseries values for multiple coordinates"() { given: def coordinates = [ - PsdmWeatherTestData.COORDINATE_193186, - PsdmWeatherTestData.COORDINATE_193187 + CosmoWeatherTestData.COORDINATE_193186, + CosmoWeatherTestData.COORDINATE_193187 ] - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval, coordinates) then: coordinateToTimeSeries.keySet().size() == 2 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186), timeSeries193186) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187), timeSeries193187) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186), timeSeries193186) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } def "A NativeSqlWeatherSource can read all weather data in a given time interval"() { given: - def timeInterval = new ClosedInterval(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.TIME_17H) + def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) def timeSeries193186 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193186_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193186_16H), - new TimeBasedValue(PsdmWeatherTestData.TIME_17H, PsdmWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193186_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193186_16H), + new TimeBasedValue(CosmoWeatherTestData.TIME_17H, CosmoWeatherTestData.WEATHER_VALUE_193186_17H)] as Set) def timeSeries193187 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193187_15H), - new TimeBasedValue(PsdmWeatherTestData.TIME_16H, PsdmWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193187_15H), + new TimeBasedValue(CosmoWeatherTestData.TIME_16H, CosmoWeatherTestData.WEATHER_VALUE_193187_16H)] as Set) def timeSeries193188 = new IndividualTimeSeries(null, [ - new TimeBasedValue(PsdmWeatherTestData.TIME_15H, PsdmWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) + new TimeBasedValue(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.WEATHER_VALUE_193188_15H)] as Set) when: Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) - equalsIgnoreUUID(coordinateToTimeSeries.get(PsdmWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeSeries193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeSeries193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeSeries193188.entries) } } diff --git a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy similarity index 98% rename from src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy rename to src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy index 4a6c95a8e..33612e176 100644 --- a/src/test/groovy/edu/ie3/test/common/PsdmWeatherTestData.groovy +++ b/src/test/groovy/edu/ie3/test/common/CosmoWeatherTestData.groovy @@ -12,7 +12,7 @@ import tech.units.indriya.quantity.Quantities import java.time.ZoneId import java.time.ZonedDateTime -class PsdmWeatherTestData extends WeatherTestData { +class CosmoWeatherTestData extends WeatherTestData { public static final ZonedDateTime TIME_15H = ZonedDateTime.of(2020, 04, 28, 15, 0, 0, 0, ZoneId.of("UTC")) public static final ZonedDateTime TIME_16H = ZonedDateTime.of(2020, 04, 28, 16, 0, 0, 0, ZoneId.of("UTC")) From 15d9550f65506727b85c524e7ab07a371efb61b8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:58:39 +0200 Subject: [PATCH 03/20] Adapt to plain naming in Couchbase files --- .../couchbase/CouchbaseWeatherSourceCosmoIT.groovy | 2 +- .../testcontainersFiles/couchbase/cosmo/weather.json | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index bff37efbf..8ea5e6976 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -37,7 +37,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinate_id" + static String coordinateIdColumnName = "coordinateid" def setupSpec() { // Copy import file with json array of documents into docker diff --git a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json index e7a49e4b2..163837bf5 100644 --- a/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json +++ b/src/test/resources/testcontainersFiles/couchbase/cosmo/weather.json @@ -1,4 +1,4 @@ -[{"coordinate_id":193186, +[{"coordinateid":193186, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":286.872985839844, "directirradiance":282.671997070312, @@ -6,7 +6,7 @@ "winddirection":0, "windvelocity":1.66103506088257}, - {"coordinate_id":193187, + {"coordinateid":193187, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":287.872985839844, "directirradiance":283.671997070312, @@ -14,7 +14,7 @@ "winddirection":0, "windvelocity":1.76103506088257}, - {"coordinate_id":193188, + {"coordinateid":193188, "time":"2020-04-28T15:00:00+00:00", "diffuseirradiance":288.872985839844, "directirradiance":284.671997070312, @@ -22,7 +22,7 @@ "winddirection":0, "windvelocity":1.86103506088257}, - {"coordinate_id":193186, + {"coordinateid":193186, "time":"2020-04-28T16:00:00+00:00", "diffuseirradiance":286.872, "directirradiance":282.672, @@ -30,7 +30,7 @@ "winddirection":0, "windvelocity":1.662}, - {"coordinate_id":193187, + {"coordinateid":193187, "time":"2020-04-28T16:00:00+00:00", "diffuseirradiance":287.872, "directirradiance":283.672, @@ -38,7 +38,7 @@ "winddirection":0, "windvelocity":1.762}, - {"coordinate_id":193186, + {"coordinateid":193186, "time":"2020-04-28T17:00:00+00:00", "diffuseirradiance":286.873, "directirradiance":282.673, From 343b832b8ac683b89e7192931a9f3bb6c0f829c0 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 18:58:56 +0200 Subject: [PATCH 04/20] Fix some tests --- .../csv/CsvWeatherSourceCosmoTest.groovy | 74 +++++++++---------- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 2 +- ...r_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv | 2 +- 3 files changed, 37 insertions(+), 41 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index aad1e856a..0f699e7c4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -5,18 +5,12 @@ */ package edu.ie3.datamodel.io.source.csv -import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy - -import static edu.ie3.datamodel.models.StandardUnits.SOLAR_IRRADIANCE -import static edu.ie3.datamodel.models.StandardUnits.TEMPERATURE -import static edu.ie3.datamodel.models.StandardUnits.WIND_DIRECTION -import static edu.ie3.datamodel.models.StandardUnits.WIND_VELOCITY - import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory -import edu.ie3.datamodel.models.value.SolarIrradianceValue +import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.SolarIrradianceValue import edu.ie3.datamodel.models.value.TemperatureValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.datamodel.models.value.WindValue @@ -31,6 +25,8 @@ import spock.lang.Shared import spock.lang.Specification import tech.units.indriya.quantity.Quantities +import static edu.ie3.datamodel.models.StandardUnits.* + class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { @Shared @@ -111,14 +107,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "5", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "5", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] def expectedValue = new TimeBasedValue( UUID.fromString("71a79f59-eebf-40c1-8358-ba7414077d57"), @@ -155,14 +151,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: @@ -180,13 +176,13 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: @@ -203,14 +199,14 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def weatherFactory = new CosmoTimeBasedWeatherValueFactory() def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ - "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", - "time" : "2020-10-16T12:40:42Z", - "coordinate" : "6", - "directirradiation" : "1.234", - "diffuseirradiation": "5.678", - "temperature" : "9.1011", - "windvelocity" : "12.1314", - "winddirection" : "15.1617" + "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", + "time" : "2020-10-16T12:40:42Z", + "coordinateid" : "6", + "directirradiance" : "1.234", + "diffuseirradiance": "5.678", + "temperature" : "9.1011", + "windvelocity" : "12.1314", + "winddirection" : "15.1617" ] when: diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index d63476667..21da4b6f5 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -35,7 +35,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest def setupSpec() { // Copy sql import script into docker - MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/psdm/weather.sql") + MountableFile sqlImportFile = MountableFile.forClasspathResource("/testcontainersFiles/sql/cosmo/weather.sql") postgreSQLContainer.copyFileToContainer(sqlImportFile, "/home/weather.sql") // Execute import script postgreSQLContainer.execInContainer("psql", "-Utest", "-f/home/weather.sql") diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv index f15c6237a..d73cac992 100644 --- a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv +++ b/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv @@ -1,4 +1,4 @@ -"uuid";"coordinate";"time";"diffuse_irradiation";"direct_irradiation";"temperature";"wind_direction";"wind_velocity" +"uuid";"coordinate_id";"time";"diffuse_irradiance";"direct_irradiance";"temperature";"wind_direction";"wind_velocity" "3cee46d5-1fe7-419e-a652-32f9be6703be";193186;2020-04-28T15:00:00Z;286.872985839844;282.671997070312;278.019012451172;0;1.66103506088257 "1fbc6f7e-52fe-4668-93c8-33454d7feb6c";193187;2020-04-28T15:00:00Z;287.872985839844;283.671997070312;279.019012451172;0;1.76103506088257 "78c7a057-ad6c-4ea1-bb74-7e28fbb08957";193188;2020-04-28T15:00:00Z;288.872985839844;284.671997070312;280.019012451172;0;1.86103506088257 From 454420dc45b16d5002937e3d5c1a460af5c4ce2d Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 6 Apr 2021 19:18:59 +0200 Subject: [PATCH 05/20] Consolidate csv weather test files in one place --- .../datamodel/io/source/csv/CsvTestDataMeta.groovy | 3 +++ .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 10 +++++----- .../io/source/csv/CsvWeatherSourceIconTest.groovy | 14 +++++--------- ...eather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv | 0 4 files changed, 13 insertions(+), 14 deletions(-) rename src/test/resources/{testTimeSeriesFiles => weather/cosmo}/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv (100%) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index 67999fad6..0bfe2cc75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -19,6 +19,9 @@ trait CsvTestDataMeta { static String gridFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("grid") static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath + static String weatherFolderPath = new File(getClass().getResource('/weather').toURI()).absolutePath + static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") + static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") static String coordinatesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("coordinates") diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 0f699e7c4..c5d7c2d7a 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -38,7 +38,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def setupSpec() { coordinateSource = CosmoWeatherTestData.coordinateSource def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -105,7 +105,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -149,7 +149,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -174,7 +174,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -197,7 +197,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> Optional.empty() def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", timeSeriesFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy index d72904c39..78c97bb75 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceIconTest.groovy @@ -29,14 +29,10 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, @Shared IdCoordinateSource coordinateSource - @Shared - String folderPath - def setupSpec() { coordinateSource = WeatherTestData.coordinateSource def weatherFactory = new IconTimeBasedWeatherValueFactory() - folderPath = new File(getClass().getResource('/weather/icon').toURI()).absolutePath - source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -106,7 +102,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 67775 ? Optional.of(expectedCoordinate) : Optional.empty() } def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) fieldToValues.putAll( [ @@ -153,7 +149,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -197,7 +193,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albRad" : "13.015240669", @@ -240,7 +236,7 @@ class CsvWeatherSourceIconTest extends Specification implements CsvTestDataMeta, given: def coordinateSource = new WeatherTestData.DummyIdCoordinateSource() def weatherFactory = new IconTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(",", folderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(",", iconWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "datum" : "2019-08-01 01:00:00", "albrad" : "13.015240669", diff --git a/src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv similarity index 100% rename from src/test/resources/testTimeSeriesFiles/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv rename to src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv From 5365a79ecc0ca62dfa9ef8c61929979d2448ab7f Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:22:28 +0200 Subject: [PATCH 06/20] Use PowerSystemUtils v 1.5.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b690f7afd..c825499de 100644 --- a/build.gradle +++ b/build.gradle @@ -52,7 +52,7 @@ repositories { dependencies { // ie³ power system utils - compile 'com.github.ie3-institute:PowerSystemUtils:1.5.1' + compile 'com.github.ie3-institute:PowerSystemUtils:1.5.2' implementation 'tech.units:indriya:2.1.2' From c9a8c161c765b64fed176df02b52fe0ff1fdf3cc Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:23:54 +0200 Subject: [PATCH 07/20] Use maven central instead of jcenter --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c825499de..92c2af400 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,7 @@ apply from: scriptsLocation + 'vcs.gradle' apply from: scriptsLocation + 'semVer.gradle' repositories { - jcenter() //searches in bintray's repository 'jCenter', which contains Maven Central + mavenCentral() // searches in Sonatype's repository 'Maven Central' maven { url 'https://www.jitpack.io' } // allows github repos as dependencies // sonatype snapshot repo From 30699e2099ae0f0044a916c4c6142537ce32a3a5 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:29:50 +0200 Subject: [PATCH 08/20] Pull naming of coordinate id {field / key / column} naming into TimeBasedWeatherValueFactory --- .../CosmoTimeBasedWeatherValueFactory.java | 6 ------ .../IconTimeBasedWeatherValueFactory.java | 6 ------ .../TimeBasedWeatherValueFactory.java | 20 +++++++++++++++++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java index 9314ab3e7..a95affade 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java @@ -26,7 +26,6 @@ * value mapping in the typical PowerSystemDataModel (PSDM) column scheme */ public class CosmoTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { - private static final String COORDINATE = "coordinateid"; private static final String DIFFUSE_IRRADIANCE = "diffuseirradiance"; private static final String DIRECT_IRRADIANCE = "directirradiance"; private static final String TEMPERATURE = "temperature"; @@ -45,11 +44,6 @@ public CosmoTimeBasedWeatherValueFactory() { super(); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java index b26c9910d..a2e23b1f2 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java @@ -29,7 +29,6 @@ */ public class IconTimeBasedWeatherValueFactory extends TimeBasedWeatherValueFactory { /* Redefine the column names to meet the icon specifications */ - private static final String COORDINATE = "coordinateid"; private static final String TIME = "datum"; private static final String DIFFUSE_IRRADIANCE = "aswdifdS"; private static final String DIRECT_IRRADIANCE = "aswdirS"; @@ -49,11 +48,6 @@ public IconTimeBasedWeatherValueFactory() { super(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd HH:mm:ss")); } - @Override - public String getCoordinateIdFieldString() { - return COORDINATE; - } - @Override public String getTimeFieldString() { return TIME; diff --git a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java index 526850763..a36b2421d 100644 --- a/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java +++ b/src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java @@ -7,6 +7,8 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.TimeUtil; +import edu.ie3.util.naming.Naming; +import edu.ie3.util.naming.NamingConvention; import java.time.ZoneId; import java.util.*; @@ -18,6 +20,8 @@ public abstract class TimeBasedWeatherValueFactory extends TimeBasedValueFactory { protected static final String UUID = "uuid"; protected static final String TIME = "time"; + /* Hold a case agnostic representation of the composite word "coordinate id", that allows for later case conversion */ + protected static final Naming COORDINATE_ID_NAMING = Naming.from("coordinate", "id"); protected final TimeUtil timeUtil; @@ -35,11 +39,23 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) { } /** - * Return the field name for the coordinate id + * Return the field name for the coordinate id in flat case. * * @return the field name for the coordinate id */ - public abstract String getCoordinateIdFieldString(); + public String getCoordinateIdFieldString() { + return COORDINATE_ID_NAMING.flatCase(); + } + + /** + * Return the field name for the coordinate id in desired case. + * + * @param convention The desired naming convention / casing + * @return the field name for the coordinate id in appropriate case + */ + public String getCoordinateIdFieldString(NamingConvention convention) { + return COORDINATE_ID_NAMING.as(convention); + } /** * Return the field name for the date time From b272635ca8f4e835b17dacd7e29afeebc26149b2 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:44:04 +0200 Subject: [PATCH 09/20] Fix InfluxDbWeatherSource --- .../influxdb/InfluxDbWeatherSource.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 62636117f..32b43e6c3 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -30,7 +30,8 @@ public class InfluxDbWeatherSource implements WeatherSource { private static final String WHERE = " where "; private static final String MEASUREMENT_NAME_WEATHER = "weather"; private static final int MILLI_TO_NANO_FACTOR = 1000000; - private final String coordinateIdColumnName; + + private final String coordinateIdFieldName; private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; @@ -50,7 +51,7 @@ public InfluxDbWeatherSource( this.connector = connector; this.coordinateSource = coordinateSource; this.weatherValueFactory = weatherValueFactory; - this.coordinateIdColumnName = weatherValueFactory.getCoordinateIdFieldString(); + this.coordinateIdFieldName = weatherValueFactory.getCoordinateIdFieldString(); } @Override @@ -152,21 +153,22 @@ private Stream>> optTimeBasedValueStream( return measurementsMap.get(MEASUREMENT_NAME_WEATHER).stream() .map( fieldToValue -> { - Optional coordinate = - coordinateSource.getCoordinate( - Integer.parseInt(fieldToValue.remove(coordinateIdColumnName))); - if (!coordinate.isPresent()) return null; - fieldToValue.putIfAbsent("uuid", UUID.randomUUID().toString()); - - /* The factory expects camel case id's for fields -> Convert the keys */ - Map camelCaseFields = + /* The factory expects flat case id's for fields -> Convert the keys */ + Map flatCaseFields = fieldToValue.entrySet().stream() .collect( Collectors.toMap( - entry -> StringUtils.snakeCaseToCamelCase(entry.getKey()), + entry -> + StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), Map.Entry::getValue)); - return new TimeBasedWeatherValueData(camelCaseFields, coordinate.get()); + Optional coordinate = + coordinateSource.getCoordinate( + Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName))); + if (!coordinate.isPresent()) return null; + flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString()); + + return new TimeBasedWeatherValueData(flatCaseFields, coordinate.get()); }) .filter(Objects::nonNull) .map(weatherValueFactory::get); From 6940eaf93294668521204d33d411c0f15f29c3c8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 08:51:07 +0200 Subject: [PATCH 10/20] Simplifying the sql weather source by fixing the column name to a given convention --- .../io/source/sql/SqlWeatherSource.java | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 8241d1c07..7a0564025 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -16,6 +16,7 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.sql.*; import java.time.ZonedDateTime; import java.util.*; @@ -31,10 +32,15 @@ public class SqlWeatherSource implements WeatherSource { private static final String DEFAULT_WEATHER_FETCHING_ERROR = "Error while fetching weather"; private static final String WHERE = " WHERE "; + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; + private final NamingConvention namingConvention; private final SqlConnector connector; private final IdCoordinateSource idCoordinateSource; - private final String factoryCoordinateFieldName; + /* The column name represents the name of the column in the chosen naming convention. However, the field name is in + * flat case always, as this is, what the factory expects. */ + private final String coordinateIdColumnName; + private final String coordinateIdFieldName; private final TimeBasedWeatherValueFactory weatherFactory; /** @@ -64,7 +70,9 @@ public SqlWeatherSource( this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.factoryCoordinateFieldName = weatherFactory.getCoordinateIdFieldString(); + this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); + this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); String dbTimeColumnName = getDbColumnName(weatherFactory.getTimeFieldString(), connector, weatherTableName) @@ -76,25 +84,15 @@ public SqlWeatherSource( + "' in provided weather data configuration." + "Please ensure that the database connection is working and the column names are correct!")); - String dbCoordColumnName = - getDbColumnName(factoryCoordinateFieldName, connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + factoryCoordinateFieldName - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); - // setup queries this.queryTimeInterval = createQueryStringForTimeInterval(schemaName, weatherTableName, dbTimeColumnName); this.queryTimeAndCoordinate = createQueryStringForTimeAndCoordinate( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); this.queryTimeIntervalAndCoordinates = createQueryStringForTimeIntervalAndCoordinates( - schemaName, weatherTableName, dbTimeColumnName, dbCoordColumnName); + schemaName, weatherTableName, dbTimeColumnName, coordinateIdColumnName); } @Override @@ -327,7 +325,7 @@ private Optional> toTimeBasedWeatherValue( */ private Optional toTimeBasedWeatherValueData( Map fieldMap) { - String coordinateValue = fieldMap.remove(factoryCoordinateFieldName); + String coordinateValue = fieldMap.remove(coordinateIdFieldName); fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString()); int coordinateId = Integer.parseInt(coordinateValue); Optional coordinate = idCoordinateSource.getCoordinate(coordinateId); From ccd9418b0d3633acba030d2b6190ea80823dcf50 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:09:47 +0200 Subject: [PATCH 11/20] Harmonizing the couchbase weather source with other source implementations --- .../source/couchbase/CouchbaseWeatherSource.java | 15 +++++++-------- .../CouchbaseWeatherSourceCosmoIT.groovy | 5 +++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 1b2d7a0d3..52cba1385 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -19,6 +19,7 @@ import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.interval.ClosedInterval; +import edu.ie3.util.naming.NamingConvention; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -33,6 +34,8 @@ /** Couchbase Source for weather data */ public class CouchbaseWeatherSource implements WeatherSource { private static final Logger logger = LogManager.getLogger(CouchbaseWeatherSource.class); + + private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT; private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; /** The start of the document key, comparable to a table name in relational databases */ private static final String DEFAULT_KEY_PREFIX = "weather"; @@ -40,6 +43,7 @@ public class CouchbaseWeatherSource implements WeatherSource { private final TimeBasedWeatherValueFactory weatherFactory; private final String keyPrefix; + private final NamingConvention namingConvention; private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; @@ -51,17 +55,14 @@ public class CouchbaseWeatherSource implements WeatherSource { * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, coordinateIdColumnName, DEFAULT_KEY_PREFIX, weatherFactory); + this(connector, coordinateSource, DEFAULT_KEY_PREFIX, weatherFactory); } /** @@ -70,8 +71,6 @@ public CouchbaseWeatherSource( * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from - * @param coordinateIdColumnName Name of the column containing the information about the - * coordinate identifier * @param keyPrefix Prefix of entries, that belong to weather * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances @@ -79,14 +78,14 @@ public CouchbaseWeatherSource( public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, - String coordinateIdColumnName, String keyPrefix, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; - this.coordinateIdColumnName = coordinateIdColumnName; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; + this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } @Override diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 8ea5e6976..1b89e8ae7 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -7,6 +7,7 @@ package edu.ie3.datamodel.io.source.couchbase import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory +import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue @@ -37,7 +38,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour @Shared CouchbaseWeatherSource source - static String coordinateIdColumnName = "coordinateid" + static String coordinateIdColumnName = TimeBasedWeatherValueFactory.COORDINATE_ID_NAMING.flatCase() def setupSpec() { // Copy import file with json array of documents into docker @@ -63,7 +64,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements WeatherSour def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, "yyyy-MM-dd'T'HH:mm:ssxxx")) - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory) + source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, weatherFactory) } def "The test container can establish a valid connection"() { From 5609e04bb91cf50c9c8c83ef65537b7aa939c1d8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:29:11 +0200 Subject: [PATCH 12/20] Offer possibility to set naming strategy in sql and couchbase weather sources --- .../couchbase/CouchbaseWeatherSource.java | 31 +++++++++++++++++-- .../io/source/sql/SqlWeatherSource.java | 31 ++++++++++++++++++- .../source/sql/SqlWeatherSourceCosmoIT.groovy | 4 ++- .../source/sql/SqlWeatherSourceIconIT.groovy | 4 ++- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 52cba1385..6ce154937 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -51,7 +51,8 @@ public class CouchbaseWeatherSource implements WeatherSource { /** * Instantiate a weather source utilising a connection to a couchbase instance obtained via the * connector. This convenient constructor uses the {@link - * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix. + * CouchbaseWeatherSource#DEFAULT_KEY_PREFIX} as key prefix and {@link + * CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. * * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from @@ -62,7 +63,29 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, TimeBasedWeatherValueFactory weatherFactory) { - this(connector, coordinateSource, DEFAULT_KEY_PREFIX, weatherFactory); + this( + connector, coordinateSource, DEFAULT_KEY_PREFIX, DEFAULT_NAMING_CONVENTION, weatherFactory); + } + + /** + * Instantiate a weather source utilising a connection to a couchbase instance obtained via the + * connector. Uses {@link CouchbaseWeatherSource#DEFAULT_NAMING_CONVENTION} as naming convention. + * + * @param connector Connector, that establishes the connection to the couchbase instance + * @param coordinateSource Source to obtain actual coordinates from + * @param keyPrefix Prefix of entries, that belong to weather + * @param weatherFactory Factory to transfer field to value mapping into actual java object + * instances + * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, + * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public CouchbaseWeatherSource( + CouchbaseConnector connector, + IdCoordinateSource coordinateSource, + String keyPrefix, + TimeBasedWeatherValueFactory weatherFactory) { + this(connector, coordinateSource, keyPrefix, DEFAULT_NAMING_CONVENTION, weatherFactory); } /** @@ -72,6 +95,7 @@ public CouchbaseWeatherSource( * @param connector Connector, that establishes the connection to the couchbase instance * @param coordinateSource Source to obtain actual coordinates from * @param keyPrefix Prefix of entries, that belong to weather + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory Factory to transfer field to value mapping into actual java object * instances */ @@ -79,12 +103,13 @@ public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, String keyPrefix, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; - this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 7a0564025..78289fe55 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -52,6 +52,33 @@ public class SqlWeatherSource implements WeatherSource { private final String queryTimeAndCoordinate; private final String queryTimeIntervalAndCoordinates; + /** + * Initializes a new SqlWeatherSource. Uses {@link SqlWeatherSource#DEFAULT_NAMING_CONVENTION} as + * naming convention. + * + * @param connector the connector needed for database connection + * @param idCoordinateSource a coordinate source to map ids to points + * @param schemaName the database schema to use + * @param weatherTableName the name of the table containing weather data + * @param weatherFactory instance of a time based weather value factory + * @deprecated Use {@link SqlWeatherSource#SqlWeatherSource(SqlConnector, IdCoordinateSource, + * String, String, NamingConvention, TimeBasedWeatherValueFactory)} instead + */ + @Deprecated + public SqlWeatherSource( + SqlConnector connector, + IdCoordinateSource idCoordinateSource, + String schemaName, + String weatherTableName, + TimeBasedWeatherValueFactory weatherFactory) { + this( + connector, + idCoordinateSource, + schemaName, + weatherTableName, + DEFAULT_NAMING_CONVENTION, + weatherFactory); + } /** * Initializes a new SqlWeatherSource * @@ -59,6 +86,7 @@ public class SqlWeatherSource implements WeatherSource { * @param idCoordinateSource a coordinate source to map ids to points * @param schemaName the database schema to use * @param weatherTableName the name of the table containing weather data + * @param namingConvention the (case) convention, how columns are named * @param weatherFactory instance of a time based weather value factory */ public SqlWeatherSource( @@ -66,11 +94,12 @@ public SqlWeatherSource( IdCoordinateSource idCoordinateSource, String schemaName, String weatherTableName, + NamingConvention namingConvention, TimeBasedWeatherValueFactory weatherFactory) { this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.namingConvention = DEFAULT_NAMING_CONVENTION; + this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy index 21da4b6f5..0b095e957 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceCosmoIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.CosmoWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.spock.Testcontainers @@ -32,6 +33,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest static String schemaName = "public" static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE def setupSpec() { // Copy sql import script into docker @@ -42,7 +44,7 @@ class SqlWeatherSourceCosmoIT extends Specification implements WeatherSourceTest def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) def weatherFactory = new CosmoTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + source = new SqlWeatherSource(connector, CosmoWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy index e91524313..4743d47d6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/sql/SqlWeatherSourceIconIT.groovy @@ -14,6 +14,7 @@ import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.TimeUtil import edu.ie3.util.interval.ClosedInterval +import edu.ie3.util.naming.NamingConvention import org.locationtech.jts.geom.Point import org.testcontainers.containers.PostgreSQLContainer import org.testcontainers.spock.Testcontainers @@ -32,6 +33,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH static String schemaName = "public" static String weatherTableName = "weather" + static NamingConvention namingConvention = NamingConvention.SNAKE def setupSpec() { // Copy sql import script into docker @@ -42,7 +44,7 @@ class SqlWeatherSourceIconIT extends Specification implements WeatherSourceTestH def connector = new SqlConnector(postgreSQLContainer.jdbcUrl, postgreSQLContainer.username, postgreSQLContainer.password) def weatherFactory = new IconTimeBasedWeatherValueFactory(TimeUtil.withDefaults) - source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, weatherFactory) + source = new SqlWeatherSource(connector, IconWeatherTestData.coordinateSource, schemaName, weatherTableName, namingConvention, weatherFactory) } def "A NativeSqlWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { From 00859be49c0040ab6bc7cc53763defb0f0254e68 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 9 Apr 2021 09:29:25 +0200 Subject: [PATCH 13/20] Document work --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e285c87e6..28ef76a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Reworking the time series source (one source per time series, distinct mapping source, factory pattern) - BREAKING: Moved methods `buildSafe{Coord,Point,LineString,LineStringBetweenCoords,LineStringBetweenPoints}`, `totalLengthOfLineString` from `GridAndGeoUtils` to `GeoUtils` in [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) - BREAKING: Moved `CoordinateDistance` to [_PowerSystemUtils_](https://github.com/ie3-institute/PowerSystemUtils) +- BREAKING: Weather source + - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`) + - Harmonized the source of coordinate id column name across implementations of `WeatherSource` + - Get field name in different casing (to actually get the column name in database, file, ...) ### Fixed - InfluxDbConnector now keeps session instead of creating a new one each call From 2c1cc77c32cfc71aef4567f4a37cd800873bbcf4 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Wed, 21 Apr 2021 11:00:42 +0200 Subject: [PATCH 14/20] Reducing code smells --- .../couchbase/CouchbaseWeatherSource.java | 2 - .../io/source/sql/SqlWeatherSource.java | 57 +------------------ 2 files changed, 2 insertions(+), 57 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 6ce154937..e44eb56e8 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -43,7 +43,6 @@ public class CouchbaseWeatherSource implements WeatherSource { private final TimeBasedWeatherValueFactory weatherFactory; private final String keyPrefix; - private final NamingConvention namingConvention; private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; private final String coordinateIdColumnName; @@ -109,7 +108,6 @@ public CouchbaseWeatherSource( this.coordinateSource = coordinateSource; this.keyPrefix = keyPrefix; this.weatherFactory = weatherFactory; - this.namingConvention = namingConvention; this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); } diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 78289fe55..bd960f42c 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -5,7 +5,6 @@ */ package edu.ie3.datamodel.io.source.sql; -import edu.ie3.datamodel.exceptions.InvalidWeatherColumnNameException; import edu.ie3.datamodel.io.connectors.SqlConnector; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueData; import edu.ie3.datamodel.io.factory.timeseries.TimeBasedWeatherValueFactory; @@ -14,7 +13,6 @@ import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries; import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue; import edu.ie3.datamodel.models.value.WeatherValue; -import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; import edu.ie3.util.naming.NamingConvention; import java.sql.*; @@ -34,12 +32,8 @@ public class SqlWeatherSource implements WeatherSource { private static final String WHERE = " WHERE "; private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; - private final NamingConvention namingConvention; private final SqlConnector connector; private final IdCoordinateSource idCoordinateSource; - /* The column name represents the name of the column in the chosen naming convention. However, the field name is in - * flat case always, as this is, what the factory expects. */ - private final String coordinateIdColumnName; private final String coordinateIdFieldName; private final TimeBasedWeatherValueFactory weatherFactory; @@ -99,19 +93,9 @@ public SqlWeatherSource( this.connector = connector; this.idCoordinateSource = idCoordinateSource; this.weatherFactory = weatherFactory; - this.namingConvention = namingConvention; - this.coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); this.coordinateIdFieldName = weatherFactory.getCoordinateIdFieldString(); - - String dbTimeColumnName = - getDbColumnName(weatherFactory.getTimeFieldString(), connector, weatherTableName) - .orElseThrow( - () -> - new InvalidWeatherColumnNameException( - "Cannot find column for '" - + weatherFactory.getTimeFieldString() - + "' in provided weather data configuration." - + "Please ensure that the database connection is working and the column names are correct!")); + String coordinateIdColumnName = weatherFactory.getCoordinateIdFieldString(namingConvention); + String dbTimeColumnName = weatherFactory.getTimeFieldString(); // setup queries this.queryTimeInterval = @@ -186,43 +170,6 @@ public Optional> getWeather(ZonedDateTime date, Poi return Optional.of(timeBasedValues.get(0)); } - /** - * Determine the corresponding database column name based on the provided factory field parameter - * name. Needed to support camel as well as snake case database column names. - * - * @param factoryColumnName the name of the field parameter set in the entity factory - * @param connector the sql connector of this source - * @param weatherTableName the table name where the weather is stored - * @return the column name that corresponds to the provided field parameter or an empty optional - * if no matching column can be found - */ - private Optional getDbColumnName( - String factoryColumnName, SqlConnector connector, String weatherTableName) { - - // get the column names from the database - Optional dbColumnName = Optional.empty(); - try { - ResultSet rs = - connector.getConnection().getMetaData().getColumns(null, null, weatherTableName, null); - - while (rs.next()) { - String databaseColumnName = rs.getString("COLUMN_NAME"); - if (StringUtils.snakeCaseToCamelCase(databaseColumnName) - .equalsIgnoreCase(factoryColumnName)) { - dbColumnName = Optional.of(databaseColumnName); - break; - } - } - } catch (SQLException ex) { - logger.error( - "Cannot connect to database to retrieve db column name for factory column name '{}' in weather table '{}'", - factoryColumnName, - weatherTableName, - ex); - } - return dbColumnName; - } - /** * Creates a base query string without closing semicolon of the following pattern:
* {@code SELECT * FROM .} From a8f1f9887611ed3d6ef350183c1bc48280fe1549 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 13:35:31 +0100 Subject: [PATCH 15/20] Fix tests --- .../datamodel/io/source/csv/CsvTestDataMeta.groovy | 2 +- .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index 4a7db7f0b..a6f1a1759 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -20,7 +20,7 @@ trait CsvTestDataMeta { static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String resultEntitiesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("results") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath - static String weatherFolderPath = new File(getClass().getResource('/weather').toURI()).absolutePath + static String weatherFolderPath = new File(CsvTestDataMeta.getResource('/weather').toURI()).absolutePath static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index c5d7c2d7a..3db1f7eda 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -7,6 +7,7 @@ package edu.ie3.datamodel.io.source.csv import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy +import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue @@ -38,7 +39,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def setupSpec() { coordinateSource = CosmoWeatherTestData.coordinateSource def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) } def "A CsvWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -105,7 +106,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -149,7 +150,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -174,7 +175,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> { args -> args[0] == 5 ? Optional.of(defaultCoordinate) : Optional.empty() } def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", @@ -197,7 +198,7 @@ class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta def coordinateSource = Mock(IdCoordinateSource) coordinateSource.getCoordinate(_) >> Optional.empty() def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new EntityPersistenceNamingStrategy(), coordinateSource, weatherFactory) + def source = new CsvWeatherSource(";", cosmoWeatherFolderPath, new FileNamingStrategy(), coordinateSource, weatherFactory) def fieldToValues = [ "uuid" : "71a79f59-eebf-40c1-8358-ba7414077d57", "time" : "2020-10-16T12:40:42Z", From 8c6b2b688b810fc47312d8e33f8f4f53877455c6 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 13:52:30 +0100 Subject: [PATCH 16/20] Improved CHANGELOG --- CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f19c018..0afdcf217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - BREAKING: Transformer's no load susceptance needs to be zero or negative to pass model validation [#378](https://github.com/ie3-institute/PowerSystemDataModel/issues/378) - All input data sets for version < 3.0.0 need to be altered! +- BREAKING: Comprehensive harmonization around weather sources [#267](https://github.com/ie3-institute/PowerSystemDataModel/issues/267) + - Adapted the expected column scheme + - DWD COSMO model + - `"coordinate"` to `"coordinate id"` (as composite word) + - `"diffuseirradiation"` to `"diffuse irradiance"` (as composite word) + - Harmonized the column name definition for coordinate id across implementations of `WeatherSource` ## [2.1.0] - 2022-01-05 @@ -55,10 +61,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - separated entity and file naming and introduced a new FileNamingStrategy taking an EntityNamingStrategy and a FileHierarchy as arguments -- BREAKING: Weather source - - Adapted data scheme (COSMO: `"coordinate"` to `"coordinate id"`, `"irradiation"` to `"irradiance"`) - - Harmonized the source of coordinate id column name across implementations of `WeatherSource` - - Get field name in different casing (to actually get the column name in database, file, ...) ### Fixed - `CsvSystemParticipantSource#getSystemParticipants()` now correctly returns electric vehicle charging station input models [PR#370](https://github.com/ie3-institute/PowerSystemDataModel/pull/370) From 6b351f014b0a45f74d8f2a130fccf2240dbef6c2 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 14:21:21 +0100 Subject: [PATCH 17/20] Improve documentation of the code --- CHANGELOG.md | 1 + .../couchbase/CouchbaseWeatherSource.java | 4 +++- .../influxdb/InfluxDbWeatherSource.java | 24 ++++++++++++------- .../io/source/sql/SqlWeatherSource.java | 3 ++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0afdcf217..c97473e1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `"coordinate"` to `"coordinate id"` (as composite word) - `"diffuseirradiation"` to `"diffuse irradiance"` (as composite word) - Harmonized the column name definition for coordinate id across implementations of `WeatherSource` + - Allows for different case conventions based on the used data source technology ## [2.1.0] - 2022-01-05 diff --git a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java index 536635cdc..85c99b53d 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java @@ -35,6 +35,7 @@ public class CouchbaseWeatherSource implements WeatherSource { private static final Logger logger = LoggerFactory.getLogger(CouchbaseWeatherSource.class); + /* Column names in couchbase come in "flat" case by default */ private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.FLAT; private static final String DEFAULT_TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ssxxx"; /** The start of the document key, comparable to a table name in relational databases */ @@ -45,6 +46,7 @@ public class CouchbaseWeatherSource implements WeatherSource { private final String keyPrefix; private final CouchbaseConnector connector; private final IdCoordinateSource coordinateSource; + /* Final name of the column within the database */ private final String coordinateIdColumnName; /** @@ -78,7 +80,7 @@ public CouchbaseWeatherSource( * @deprecated Use {@link CouchbaseWeatherSource#CouchbaseWeatherSource(CouchbaseConnector, * IdCoordinateSource, String, NamingConvention, TimeBasedWeatherValueFactory)} instead */ - @Deprecated + @Deprecated(since = "3.0", forRemoval = true) public CouchbaseWeatherSource( CouchbaseConnector connector, IdCoordinateSource coordinateSource, diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 32b43e6c3..863110c0e 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -15,10 +15,13 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; + import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; + +import edu.ie3.util.naming.Naming; import org.influxdb.InfluxDB; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; @@ -31,7 +34,10 @@ public class InfluxDbWeatherSource implements WeatherSource { private static final String MEASUREMENT_NAME_WEATHER = "weather"; private static final int MILLI_TO_NANO_FACTOR = 1000000; + /* Final name of the coordinate id field for use in factories */ private final String coordinateIdFieldName; + /* Final name of the column within the database */ + private final String coordinateIdColumnName = Naming.from("coordinate", "id").snakeCase(); private final InfluxDbConnector connector; private final IdCoordinateSource coordinateSource; private final TimeBasedWeatherValueFactory weatherValueFactory; @@ -162,16 +168,16 @@ private Stream>> optTimeBasedValueStream( StringUtils.snakeCaseToCamelCase(entry.getKey()).toLowerCase(), Map.Entry::getValue)); - Optional coordinate = - coordinateSource.getCoordinate( - Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName))); - if (!coordinate.isPresent()) return null; + /* Add a random UUID if necessary */ flatCaseFields.putIfAbsent("uuid", UUID.randomUUID().toString()); - return new TimeBasedWeatherValueData(flatCaseFields, coordinate.get()); - }) - .filter(Objects::nonNull) - .map(weatherValueFactory::get); + /* Get the corresponding coordinate id from map AND REMOVE THE ENTRY !!! */ + int coordinateId = Integer.parseInt(flatCaseFields.remove(coordinateIdFieldName)); + return coordinateSource + .getCoordinate(coordinateId) + .map(point -> new TimeBasedWeatherValueData(flatCaseFields, point)) + .flatMap(weatherValueFactory::get); + }); } private String createQueryStringForCoordinateAndTimeInterval( @@ -207,7 +213,7 @@ private String createTimeConstraint(ZonedDateTime date) { } private String createCoordinateConstraintString(int coordinateId) { - return "coordinate_id='" + coordinateId + "'"; + return coordinateIdColumnName + "='" + coordinateId + "'"; } /** diff --git a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java index 83fe94945..df7f6fb64 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/sql/SqlWeatherSource.java @@ -25,6 +25,7 @@ public class SqlWeatherSource extends SqlDataSource> implements WeatherSource { private static final String WHERE = " WHERE "; + /* Column names in sql come in "snake" case by default */ private static final NamingConvention DEFAULT_NAMING_CONVENTION = NamingConvention.SNAKE; private final IdCoordinateSource idCoordinateSource; @@ -52,7 +53,7 @@ public class SqlWeatherSource extends SqlDataSource * @deprecated Use {@link SqlWeatherSource#SqlWeatherSource(SqlConnector, IdCoordinateSource, * String, String, NamingConvention, TimeBasedWeatherValueFactory)} instead */ - @Deprecated + @Deprecated(since = "3.0", forRemoval = true) public SqlWeatherSource( SqlConnector connector, IdCoordinateSource idCoordinateSource, From a6d8fdd02ed5adfa95abf73ecb8e9700a3800159 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 14:28:52 +0100 Subject: [PATCH 18/20] Move test data to location within the package structure --- .../edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy | 2 +- .../cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv | 0 .../icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename src/test/resources/{ => edu/ie3/datamodel/io/source/csv}/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv (100%) rename src/test/resources/{ => edu/ie3/datamodel/io/source/csv}/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv (100%) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy index a6f1a1759..f3c5459e3 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvTestDataMeta.groovy @@ -20,7 +20,7 @@ trait CsvTestDataMeta { static String participantsFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("participants") static String resultEntitiesFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("results") static String timeSeriesFolderPath = testTimeSeriesBaseFolderPath - static String weatherFolderPath = new File(CsvTestDataMeta.getResource('/weather').toURI()).absolutePath + static String weatherFolderPath = new File(CsvTestDataMeta.getResource('weather').toURI()).absolutePath static String cosmoWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("cosmo") static String iconWeatherFolderPath = weatherFolderPath.concat(File.separator).concat("icon") static String thermalFolderPath = testParticipantsBaseFolderPath.concat(File.separator).concat("thermal") diff --git a/src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv similarity index 100% rename from src/test/resources/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv rename to src/test/resources/edu/ie3/datamodel/io/source/csv/weather/cosmo/its_weather_8bc9120d-fb9b-4484-b4e3-0cdadf0feea9.csv diff --git a/src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv b/src/test/resources/edu/ie3/datamodel/io/source/csv/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv similarity index 100% rename from src/test/resources/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv rename to src/test/resources/edu/ie3/datamodel/io/source/csv/weather/icon/its_weather_513606bc-539e-445b-9675-2f98be3d9231.csv From da408e1fa56743e1317f4338519e28cc09da2bc3 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 14:33:24 +0100 Subject: [PATCH 19/20] Fmt --- .../datamodel/io/source/influxdb/InfluxDbWeatherSource.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java index 863110c0e..c1f7d5d57 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSource.java @@ -15,13 +15,11 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.util.StringUtils; import edu.ie3.util.interval.ClosedInterval; - +import edu.ie3.util.naming.Naming; import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; - -import edu.ie3.util.naming.Naming; import org.influxdb.InfluxDB; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; From 4dfbe0cd2113e894a6add0035a86a0df56730cb6 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 28 Jan 2022 15:02:22 +0100 Subject: [PATCH 20/20] reducing code smells --- .../io/source/csv/CsvWeatherSourceCosmoTest.groovy | 5 ++--- .../source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy | 9 +++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy index 3db1f7eda..b93cdbf11 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/csv/CsvWeatherSourceCosmoTest.groovy @@ -5,8 +5,9 @@ */ package edu.ie3.datamodel.io.source.csv +import static edu.ie3.datamodel.models.StandardUnits.* + import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory -import edu.ie3.datamodel.io.naming.EntityPersistenceNamingStrategy import edu.ie3.datamodel.io.naming.FileNamingStrategy import edu.ie3.datamodel.io.source.IdCoordinateSource import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries @@ -26,8 +27,6 @@ import spock.lang.Shared import spock.lang.Specification import tech.units.indriya.quantity.Quantities -import static edu.ie3.datamodel.models.StandardUnits.* - class CsvWeatherSourceCosmoTest extends Specification implements CsvTestDataMeta, WeatherSourceTestHelper { @Shared diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy index 0d37e014f..2c5a2b5b4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/influxdb/InfluxDbWeatherSourceCosmoIT.groovy @@ -17,6 +17,7 @@ import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point import org.testcontainers.containers.InfluxDBContainer import org.testcontainers.spock.Testcontainers +import org.testcontainers.utility.DockerImageName import org.testcontainers.utility.MountableFile import spock.lang.Shared import spock.lang.Specification @@ -25,7 +26,7 @@ import spock.lang.Specification class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourceTestHelper { @Shared - InfluxDBContainer influxDbContainer = new InfluxDBContainer("1.8.4") + InfluxDBContainer influxDbContainer = new InfluxDBContainer(DockerImageName.parse("influxdb:1.8.10")) .withAuthEnabled(false) .withDatabase("test_weather") @@ -112,9 +113,9 @@ class InfluxDbWeatherSourceCosmoIT extends Specification implements WeatherSourc Map> coordinateToTimeSeries = source.getWeather(timeInterval) then: coordinateToTimeSeries.keySet().size() == 3 - equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).getEntries(), timeseries_193186.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).getEntries(), timeseries_193187.getEntries()) - equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).getEntries(), timeseries_193188.getEntries()) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193186).entries, timeseries_193186.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187).entries, timeseries_193187.entries) + equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193188).entries, timeseries_193188.entries) } def "An InfluxDbWeatherSource will return an equivalent to 'empty' when being unable to map a coordinate to it's ID"() {