diff --git a/Dockerfile b/Dockerfile index f85ec4c34..02d9b209d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,6 +79,7 @@ ENV cwms.dataapi.access.openid.timeout="604800" ENV cwms.dataapi.access.openid.clientId=cwms ENV cwms.dataapi.access.openid.idpHint=federation-eams #ENV cwms.dataapi.access.openid.altAuthUrl="https://identityc-test.cwbi.us/auth/realms/cwbi" +ENV TS_USE_DMZ_INCLUDE="true" # used to simplify redeploy in certain contexts. Update to match - in image label ENV IMAGE_MARKER="a" diff --git a/cwms-data-api/build.gradle b/cwms-data-api/build.gradle index 9225da419..c927f61dc 100644 --- a/cwms-data-api/build.gradle +++ b/cwms-data-api/build.gradle @@ -269,7 +269,7 @@ task run(type: JavaExec) { systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") } systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") } systemProperties += project.properties.findAll { k, v -> k.startsWith("cwms") } - + environment("TS_USE_DMZ_INCLUDE", "true") def context = project.findProperty("cda.war.context") ?: "spk-data" args "$buildDir/tomcat", "$buildDir/libs/${project.name}-${project.version}.war", context @@ -319,6 +319,9 @@ task integrationTests(type: Test) { systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") && !k.startsWith("RADAR_JDBC") } systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") && !k.startsWith("CDA_JDBC") } systemProperties += project.properties.findAll { k, v -> k.startsWith("testcontainer") } + + environment("TS_USE_DMZ_INCLUDE", "true") + jvmArgs += "-DwarFile=$buildDir/libs/${project.name}-${project.version}.war" jvmArgs += "-DwarContext=/cwms-data" jvmArgs += "-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager" diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index f5871a278..c15adf78f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -1,6 +1,7 @@ package cwms.cda.api; import static com.codahale.metrics.MetricRegistry.name; +import static cwms.cda.ApiServlet.CWMS_USERS_ROLE; import static cwms.cda.api.Controllers.*; import com.codahale.metrics.Histogram; @@ -29,6 +30,9 @@ import cwms.cda.data.dto.csv.TimeSeriesCsv; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DateUtils; +import cwms.cda.security.CdaAccessManager; +import cwms.cda.security.CwmsAuthException; +import cwms.cda.security.Role; import io.javalin.apibuilder.CrudHandler; import io.javalin.core.util.Header; import io.javalin.core.validation.JavalinValidation; @@ -41,6 +45,9 @@ import io.javalin.plugin.openapi.annotations.OpenApiParam; import io.javalin.plugin.openapi.annotations.OpenApiRequestBody; import io.javalin.plugin.openapi.annotations.OpenApiResponse; +import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; +import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN; + import java.io.IOException; import java.io.StringWriter; import java.net.URISyntaxException; @@ -52,11 +59,13 @@ import java.util.concurrent.TimeUnit; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpStatus; import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URLEncodedUtils; import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; public class TimeSeriesController implements CrudHandler { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -121,6 +130,8 @@ public class TimeSeriesController implements CrudHandler { private final Histogram requestResultSize; static final int DEFAULT_PAGE_SIZE = 500; + private static final boolean USE_DMZ_LIST = Boolean.parseBoolean(System.getenv("TS_USE_DMZ_INCLUDE")); + public TimeSeriesController(MetricRegistry metrics) { this.metrics = metrics; @@ -445,7 +456,6 @@ public void getAll(@NotNull Context ctx) { TimeSeriesDao dao = getTimeSeriesDao(dsl); String format = ctx.queryParamAsClass(FORMAT, String.class).getOrDefault(""); String names = requiredParam(ctx, NAME); - //try 'unit' if 'units' is not provided as that was the original parameter name String units = ctx.queryParamAsClass(UNITS, String.class) .getOrDefault(ctx.queryParamAsClass(UNIT, String.class) @@ -498,7 +508,7 @@ public void getAll(@NotNull Context ctx) { ZonedDateTime endZdt = end != null ? DateUtils.parseUserDate(end, timezone) : ZonedDateTime.now(tz); - + checkAuth(ctx, names, ctx.queryParam(OFFICE)); if (version != null && version.equals("2")) { String office = requiredParam(ctx, OFFICE); @@ -620,6 +630,59 @@ public void getAll(@NotNull Context ctx) { } } + /** + * Presumably a stop gap until formal authorization is in place. Compares provided time series id to DMZ include TS + * Group and if the session is authorized to determine if the request should proceed. + * @param ctx + * @param dsl + * @param names + * @param office + */ + public static void checkAuth(Context ctx, String names, String office) + { + if (!USE_DMZ_LIST) + { + return; + } + final var authEx = new CwmsAuthException("Required time series is not authorized for public release at this time.", HttpStatus.SC_UNAUTHORIZED); + var p = CdaAccessManager.getApiPrincipal(ctx); + if (p != null && p.getRoles().contains(new Role(CWMS_USERS_ROLE))) + { + logger.atInfo().log("access allowed, authenticated. %s", p.getName()); + return; // at this time there are no access restrictions on "CWMS Users" + } + final var TS_ID = AV_CWMS_TS_ID.AV_CWMS_TS_ID; + final var TS_GROUP = AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN; + var dsl = JooqDao.getDslContext(ctx, null); + var countQuery = + dsl.selectOne() + .from(TS_GROUP) + .where( + TS_GROUP.GROUP_ID.eq("DMZ Include List") + .and(TS_GROUP.TS_CODE.in( + dsl.select(TS_ID.TS_CODE) + .from(TS_ID) + // There are a few endpoints which may not required the office id + // with a time series id. in that case office will be null, possibly + // restricting data when it should not be. Unfortunately, as their may + // be duplicate names, there is not a better way to handle that beside + // requiring the office id, which is the behavior of any new endpoint. + .where(TS_ID.DB_OFFICE_ID.equalIgnoreCase(office).and( + TS_ID.CWMS_TS_ID.equalIgnoreCase(names))) + ) + ) + ); + boolean found = dsl.fetchExists(countQuery); + + if (found) + { + logger.atInfo().log("access allowed, include list."); + return; + } + logger.atInfo().log("access denied, include list."); + throw authEx; + } + private void streamCsv(@NotNull Context ctx, CsvConfiguration csvConfig, int batchSize, TimeSeriesDao dao, TimeSeriesRequestParameters requestParameters) { int csvBatchSize = validateCsvBatchSize(batchSize); dao.streamRequestedTimeSeriesCsv( diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java index ea144fe66..724bd15d2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java @@ -217,7 +217,7 @@ public void handle(@NotNull Context ctx) { : ZonedDateTime.now(tz); String office = ctx.queryParam(OFFICE); - + TimeSeriesController.checkAuth(ctx, names, office); FilteredTimeSeriesParameters ftsParams = FilteredTimeSeriesParameters.Builder.from(ctx) .build(); diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java index 39d4895ef..7ae32f6b2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java @@ -49,6 +49,7 @@ import cwms.cda.data.dto.RecentValue; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; +import cwms.cda.security.CwmsAuthException; import io.javalin.core.util.Header; import io.javalin.http.Context; import io.javalin.http.Handler; @@ -125,7 +126,9 @@ private TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { + "implemented") }, path = "/timeseries/recent", - description = "Returns CWMS Timeseries Groups Data", + description = "Returns CWMS Timeseries Groups Data. NOTE: as groups may be defined with restricted time series" + + "for internal use, instead of the whole request returning a 403 error the restricted data is" + + "instead silently removed so that rest of the group may be retrieved.", tags = TimeSeriesController.TAG, method = HttpMethod.GET ) @@ -181,6 +184,21 @@ public void handle(@NotNull Context ctx) { } else { latestValues = dao.findMostRecentsInRange(office, tsIds, pastLimit, futureLimit, unitSystem); } + latestValues = latestValues.stream() + .filter(rv -> { + // while not the most performant code, + // it'll work. Additionally for users with authorization + // checkAuth bypasses any database calls + try { + TimeSeriesController.checkAuth(ctx, + rv.getDqu().getCwmsTsId(), + rv.getDqu().getOfficeId()); + return true; + } catch (CwmsAuthException ex) { + return false; + } + }) + .collect(Collectors.toList()); String formatHeader = ctx.header(Header.ACCEPT); ContentType contentType = Formats.parseHeader(formatHeader, RecentValue.class); diff --git a/cwms-data-api/src/main/java/cwms/cda/security/CdaAccessManager.java b/cwms-data-api/src/main/java/cwms/cda/security/CdaAccessManager.java index bb3b558b0..7062a8fa9 100644 --- a/cwms-data-api/src/main/java/cwms/cda/security/CdaAccessManager.java +++ b/cwms-data-api/src/main/java/cwms/cda/security/CdaAccessManager.java @@ -68,7 +68,7 @@ private void checkRateLimit(Context ctx) { } } - private DataApiPrincipal getApiPrincipal(Context ctx) { + public static DataApiPrincipal getApiPrincipal(Context ctx) { return ctx.sessionAttribute(IdentityProvider.PRINCIPAL_KEY); } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/AccessManagerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/AccessManagerTestIT.java index 4e4dffeeb..2c4ad7d3c 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/AccessManagerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/AccessManagerTestIT.java @@ -5,6 +5,7 @@ import fixtures.TestAccounts; import fixtures.users.UserSpecSource; import fixtures.users.annotation.AuthType; +import fixtures.users.annotation.AuthType.UserType; import io.restassured.filter.log.LogDetail; import io.restassured.specification.RequestSpecification; @@ -60,6 +61,58 @@ void cant_create_without_user(String authType, TestAccounts.KeyUser user, Reques .statusCode(is(HttpServletResponse.SC_UNAUTHORIZED)); } + @ParameterizedTest + @ArgumentsSource (UserSpecSource.class) + @AuthType(userTypes = {UserType.GUEST_AND_PRIVS}) + void test_ts_auth_not_in_dmz_list(String authType, TestAccounts.KeyUser user, RequestSpecification authSpec) throws Exception + { + + final String tsName = "Restricted.Stage.Inst.1Hour.0.AuthTest"; + final String office = user.getOperatingOffice() == null ? "SPK" : user.getOperatingOffice(); + createLocation("Restricted", true, office); + createTimeseries(office, tsName, false); + given() + .log().ifValidationFails(LogDetail.ALL,true) + .spec(authSpec) + .contentType("application/json") + .queryParam("office", office) + .queryParam("name", tsName) + .when() + .get( "/timeseries") + .then() + .assertThat() + .log().ifValidationFails(LogDetail.ALL,true) + .statusCode(is( + user == TestAccounts.KeyUser.GUEST || + user == TestAccounts.KeyUser.NONE || + user == TestAccounts.KeyUser.SPK_CAC_BUT_NOT_CWMS_USER + ? HttpServletResponse.SC_UNAUTHORIZED : HttpServletResponse.SC_OK)); + } + + @ParameterizedTest + @ArgumentsSource (UserSpecSource.class) + @AuthType(userTypes = {UserType.GUEST_AND_PRIVS}) + void test_ts_auth_in_dmz_list(String authType, TestAccounts.KeyUser user, RequestSpecification authSpec) throws Exception + { + + final String tsName = "NotRestricted.Stage.Inst.1Hour.0.AuthTest"; + final String office = user.getOperatingOffice() == null ? "SPK" : user.getOperatingOffice(); + createLocation("NotRestricted", true, office); + createTimeseries(office, tsName, true); + given() + .log().ifValidationFails(LogDetail.ALL,true) + .spec(authSpec) + .contentType("application/json") + .queryParam("office", office) + .queryParam("name", tsName) + .when() + .get( "/timeseries") + .then() + .assertThat() + .log().ifValidationFails(LogDetail.ALL,true) + .statusCode(is(HttpServletResponse.SC_OK)); + } + @ParameterizedTest @ArgumentsSource(UserSpecSource.class) @AuthType(userTypes = { AuthType.UserType.PRIVS }) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java index 036e97cc9..d5c8c4b7f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java @@ -490,6 +490,19 @@ protected static void deleteLocation(String location, String office) throws SQLE * @throws SQLException */ protected static void createTimeseries(String office, String timeseries) throws SQLException { + createTimeseries(office, timeseries, true); + } + + /** + * Create a timeseries (location must already exist), no data or other meta data will be set. + * This only creates the timeseries name. Not data or other parameters are set. + * + * @param office owning office + * @param timeseries timeseries name + * @param inDmzList if this timeseries id should go in the DMZ include list. + * @throws SQLException + */ + protected static void createTimeseries(String office, String timeseries, boolean inDmzList) throws SQLException { CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); db.connection((c) -> { try (PreparedStatement stmt = c.prepareStatement(createTimeseriesQuery)) { @@ -503,6 +516,36 @@ protected static void createTimeseries(String office, String timeseries) throws throw new RuntimeException("Unable to create timeseries", ex); } }, "cwms_20"); + if (inDmzList) { + assignTsToDmzInclude(office, timeseries); + } + } + + /** + * Stop gap to make establishing timeseries for tests easier + * without disabling the time series auth check. + * NOTE: this method currently assumes the database default is still using ~ for local regular + * instead of the new intervals with Local in the name. It will eventually need to be adjusted. + * Though preferably as this is part of a stop gap change, this method would just go away. + * @param office + * @param timeSeries + * @throws SQLException + */ + protected static void assignTsToDmzInclude(String office, String timeSeries) throws SQLException { + CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); + final String query = "begin cwms_ts.assign_ts_group(p_ts_category_id => 'Data Dissemination', p_ts_group_id => 'DMZ Include List', p_ts_id => ?, p_db_office_id => ?); end;"; + String ts = timeSeries.replaceFirst("(\\d+.*?)Local", "~$1"); + logger.atInfo().log("Replacing %s with %s", timeSeries, ts); + db.connection((c) -> { + try (PreparedStatement stmt = c.prepareStatement(query)) { + // at this time the default is using non-local names + stmt.setString(1, ts); + stmt.setString(2, office); + stmt.execute(); + } catch (SQLException ex) { + throw new RuntimeException("Unable to create timeseries", ex); + } + }, "cwms_20"); } @@ -521,6 +564,7 @@ protected static void createTimeseries(String office, String timeseries, int off throw new RuntimeException("Unable to create timeseries", ex); } }, "cwms_20"); + assignTsToDmzInclude(office, timeseries); } protected static void createTimeseriesWithNewLRTSInterval(String office, String timeseries, int offset) throws SQLException { @@ -541,6 +585,7 @@ protected static void createTimeseriesWithNewLRTSInterval(String office, String throw new RuntimeException("Unable to create timeseries", ex); } }, "cwms_20"); + assignTsToDmzInclude(office, timeseries); } /** diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java index ca2bb21cf..48f8ad2ce 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java @@ -66,7 +66,7 @@ void test_page_size_special_cases(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) .log().ifValidationFails(LogDetail.ALL,true) @@ -147,7 +147,7 @@ void test_filter_nulls(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back without filtering given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -236,7 +236,7 @@ void test_min_value(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back with min-value given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -299,7 +299,7 @@ void test_max_value(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back with max-value given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -362,7 +362,7 @@ void test_min_max_value_combined(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back with min-value and max-value given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -425,7 +425,7 @@ void test_all_filters_combined(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back with filter-nulls, min-value, and max-value given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java index 16e1ced44..70c8049d9 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesRecentControllerIT.java @@ -77,6 +77,7 @@ class TimeSeriesRecentControllerIT extends DataApiTestIT { private static final String OFFICE_ID = "SPK"; private static final String LOCATION = "Sacramento River Delta"; private static final String TS_ID = LOCATION + ".Depth.Inst.15Minutes.0.OBS-Raw"; + private static final String TS_ID_RESTRICTED = LOCATION + ".Depth.Inst.15Minutes.0.OBS-Raw-Restricted"; private static final String CATEGORY_ID = "USACE Data Acquisition"; private static final String GROUP_ID = "USACE Include List"; private static final ZonedDateTime END = ZonedDateTime.now(); @@ -228,6 +229,160 @@ void test_retrieving_recent_ts_data(String format) throws Exception { ; } + + @ParameterizedTest + @ValueSource(strings = {Formats.JSONV1, Formats.DEFAULT}) + void test_retrieving_recent_ts_data_mixed_auth(String format) throws Exception { + TimeSeries ts = buildTimeSeries(OFFICE_ID, TS_ID); + TimeSeries tsIdRestricted = buildTimeSeries(OFFICE_ID, TS_ID_RESTRICTED); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + String json = Formats.format(contentType, ts); + String jsonRestricted = Formats.format(contentType, tsIdRestricted); + + // create timeseries + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV2) + .accept(Formats.JSONV2) + .header(AUTH_HEADER, user.toHeaderValue()) + .body(json) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + ; + + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV2) + .accept(Formats.JSONV2) + .header(AUTH_HEADER, user.toHeaderValue()) + .body(jsonRestricted) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + ; + assignTsToDmzInclude(ts.getOfficeId(), ts.getName()); + + TimeSeriesCategory category = new TimeSeriesCategory(OFFICE_ID, CATEGORY_ID, "Data Acquisition category"); + json = JsonV1.buildObjectMapper().writeValueAsString(category); + + // add timeseries to category + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .accept(format) + .header(AUTH_HEADER, user.toHeaderValue()) + .queryParam(FAIL_IF_EXISTS, false) + .body(json) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/category") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_CREATED)) + ; + + group = new TimeSeriesGroup(category, OFFICE_ID, GROUP_ID, "USACE Include group", null, TS_ID); + List tsList = List.of( + new AssignedTimeSeries(OFFICE_ID, TS_ID, null, TS_ID, 0), + new AssignedTimeSeries(OFFICE_ID, TS_ID_RESTRICTED, null, TS_ID_RESTRICTED, 0) + ); + group = new TimeSeriesGroup(group, tsList); + json = JsonV1.buildObjectMapper().writeValueAsString(group); + + // add timeseries to group + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .accept(format) + .header(AUTH_HEADER, user.toHeaderValue()) + .queryParam(FAIL_IF_EXISTS, false) + .body(json) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/group") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_CREATED)) + ; + + // get recent data using category and group + // with authorization + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .accept(format) + .header(AUTH_HEADER, user.toHeaderValue()) + .queryParam(OFFICE, OFFICE_ID) + .queryParam(Controllers.CATEGORY_ID, CATEGORY_ID) + .queryParam(Controllers.GROUP_ID, GROUP_ID) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/recent/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("size()", is(2)) + .time(lessThan(5000L)) + ; + + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .accept(format) + .queryParam(OFFICE, OFFICE_ID) + .queryParam(Controllers.CATEGORY_ID, CATEGORY_ID) + .queryParam(Controllers.GROUP_ID, GROUP_ID) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/recent/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("size()", is(1)) + .time(lessThan(5000L)) + ; + + // get recent data using timeseries id + given() + .log().ifValidationFails(LogDetail.ALL, true) + .contentType(Formats.JSONV1) + .accept(format) + .header(AUTH_HEADER, user.toHeaderValue()) + .queryParam(OFFICE, OFFICE_ID) + .queryParam(TS_IDS, TS_ID) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/recent/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("size()", is(1)) + .time(lessThan(5000L)) + ; + } + + @NotNull private TimeSeries buildTimeSeries(String officeId, String tsId) { long diff = END.toEpochSecond() - START.toEpochSecond(); diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index f2de16bcb..ada228fd6 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -110,6 +110,8 @@ void test_lrl_timeseries_psuedo_reg1hour() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); + assignTsToDmzInclude(officeId, ts.get("name").asText()); + // get it back given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -177,6 +179,8 @@ void test_local_regular_new_LRTS_ID() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); + assignTsToDmzInclude(officeId, ts.get("name").asText().replace("1DayLocal", "~1Day")); + // get it back given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -280,6 +284,7 @@ void test_lrl_1day() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 @@ -344,6 +349,7 @@ void test_lrl_1day_default() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 @@ -405,7 +411,7 @@ void test_lrl_1day_max_version() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String secondVersionDate = "1604786000000"; tsData = tsData.replace("1594786000000", secondVersionDate).replace("35,", "47.5,"); // inserting the second time series @@ -513,7 +519,7 @@ void test_lrl_1day_max_version_with_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String secondVersionDate = "1604786000000"; tsData = tsData.replace("1594786000000", secondVersionDate).replace("35,", "47.5,"); // inserting the second time series @@ -661,7 +667,7 @@ void test_lrl_timeseries_psuedo_reg1week() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -743,7 +749,7 @@ void test_lrl_timeseries_lrts_reg1week() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -841,7 +847,7 @@ void test_include_data_entry_date() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) ; - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // 1675335600000 is Thursday, February 2, 2023 11:00:00 AM // fyi 1675422000000 is Friday, February 3, 2023 11:00:00 AM @@ -928,7 +934,7 @@ void test_get_with_units() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) ; - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back using 'units' parameter given() .log().ifValidationFails(LogDetail.ALL, true) @@ -1109,7 +1115,7 @@ void test_delete_ts() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -1192,6 +1198,7 @@ void test_no_office_permissions() throws Exception { @Test void test_invalid_office() { + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -1199,6 +1206,7 @@ void test_invalid_office() { //Purposefully misspelled office id .queryParam(OFFICE, "NWDW") .queryParam(NAME, "Buckhorn.Temp-Water.Inst.1Day.0.cda-test") + .header("Authorization", user.toHeaderValue()) .when() .redirects().follow(true) .redirects().max(3) @@ -1299,7 +1307,7 @@ void test_v2_can_datum() throws Exception { String location = ts.get(NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); createLocation(location, true, officeId); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 given() @@ -1358,7 +1366,7 @@ void test_lrl_trim() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // The ts we created has two values 1675335600000, 1675422000000, // get it back @@ -1457,7 +1465,7 @@ void test_lrl_trim_with_data_entry_date() throws Exception { .log().ifValidationFails(LogDetail.ALL, true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // The ts we created has two values 1675335600000, 1675422000000, @@ -1667,7 +1675,7 @@ void test_daylight_saving_retrieve()throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)) ; - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // this doesn't cross Daylight savings - should work given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -1749,7 +1757,7 @@ void test_lrl_1day_content_type_aliasing(GetAllTest test) throws Exception .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back given() .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) @@ -1804,7 +1812,7 @@ void test_wrong_units() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // get it back String firstPoint = "2023-02-02T06:00:00-05:00"; //aka 2023-02-02T11:00:00.000Z or // 1675335600000 @@ -1859,7 +1867,7 @@ void test_csv_default_columns_with_metadata() throws Exception { .then() .log().ifValidationFails(LogDetail.ALL,true) .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // retrieve CSV v2 with metadata comments explicitly requested String body = given() .log().ifValidationFails(LogDetail.ALL, true) @@ -1909,7 +1917,7 @@ void test_csv_optional_columns_no_metadata() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String body = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.CSV) @@ -1956,7 +1964,7 @@ void test_csv_optional_columns_with_metadata() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String body = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.CSV) @@ -2013,7 +2021,7 @@ void test_csv_streaming_no_missing_data() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String secondVersionDate = "1604786000000"; tsData = tsData.replace("1594786000000", secondVersionDate).replace("35,", "47.5,"); // inserting the second time series @@ -2125,7 +2133,7 @@ void test_csv_streaming_with_optionals() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // 2. Client requests CSV with optional columns and receives a single streamed response. String body = given() .log().ifValidationFails(LogDetail.ALL, true) @@ -2186,7 +2194,7 @@ void test_csv_streaming() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // 2. Client requests CSV and receives a single streamed response. // The small page-size (10) forces the server to perform 5 internal fetch cycles // for the 50 points in large_timeseries.json. @@ -2304,7 +2312,7 @@ void test_query_param_format_overrides_header() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); //this should use csv not json because the query param overrides the header String body = given() .log().ifValidationFails(LogDetail.ALL, true) @@ -2360,7 +2368,7 @@ void test_csv_default_columns_no_metadata() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String body = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.CSV) @@ -2406,7 +2414,7 @@ void test_csv_date_format_query_param() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String firstPoint = "2023-01-11T12:00:00Z"; // Test with date-format param in query string @@ -2455,7 +2463,7 @@ void test_csv_date_format_custom_pattern() throws Exception { .post("/timeseries/") .then() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); String firstPoint = "2023-01-11T12:00:00Z"; // Test with custom date-format-pattern @@ -2534,7 +2542,7 @@ void test_get_for_elev_has_datum() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(officeId, ts.get("name").asText()); // 1209654000000 as ms == Thursday, May 1, 2008 3:00:00 PM @@ -2738,7 +2746,6 @@ void test_create_without_vertical_datum_info() throws Exception { createLocation(location, true, officeId); // This marks for delete at end of test. updateLocation(location, true, officeId); - TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; // Helper lambda to GET the series and return a ValidatableResponse @@ -2752,6 +2759,7 @@ void test_create_without_vertical_datum_info() throws Exception { .queryParam(BEGIN, beginIso) .queryParam(END, endIso) .queryParam(TRIM, true) + .header("Authorization", user.toHeaderValue()) .when() .redirects().follow(true) .redirects().max(3) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/VersionedTimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/VersionedTimeseriesControllerTestIT.java index 630d6a184..c030c55ff 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/VersionedTimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/VersionedTimeseriesControllerTestIT.java @@ -74,7 +74,7 @@ void test_create_get_delete_get_versioned(String format) throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(ts.getOfficeId(), ts.getName()); // Get versioned time series given() .log().ifValidationFails(LogDetail.ALL,true) @@ -178,7 +178,7 @@ void test_create_get_update_get_delete_get_versioned(String format) throws Excep .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(ts.getOfficeId(), ts.getName()); // Get versioned time series given() @@ -330,7 +330,7 @@ void test_create_get_update_get_delete_get_unversioned(String format) throws Exc .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(ts.getOfficeId(), ts.getName()); // Get versioned time series given() .log().ifValidationFails(LogDetail.ALL,true) @@ -484,7 +484,7 @@ void test_create_get_update_get_delete_get_max_agg(String format) throws Excepti .log().ifValidationFails(LogDetail.ALL,true) .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - + assignTsToDmzInclude(ts.getOfficeId(), ts.getName()); given() .log().ifValidationFails(LogDetail.ALL,true) .accept(format)