Establish incredibly simple time series auth restriction using the ex…

Dockerfile

@@ -79,6 +79,7 @@ ENV cwms.dataapi.access.openid.timeout="604800"
 ENV cwms.dataapi.access.openid.clientId=cwms
 ENV cwms.dataapi.access.openid.idpHint=federation-eams
 #ENV cwms.dataapi.access.openid.altAuthUrl="https://identityc-test.cwbi.us/auth/realms/cwbi"
+ENV TS_USE_DMZ_INCLUDE="true"
 
 # used to simplify redeploy in certain contexts. Update to match -<marker> in image label
 ENV IMAGE_MARKER="a"

cwms-data-api/build.gradle

@@ -269,7 +269,7 @@ task run(type: JavaExec) {
     systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") }
     systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") }
     systemProperties += project.properties.findAll { k, v -> k.startsWith("cwms") }
-
+    environment("TS_USE_DMZ_INCLUDE", "true")
     def context = project.findProperty("cda.war.context") ?: "spk-data"
 
     args "$buildDir/tomcat", "$buildDir/libs/${project.name}-${project.version}.war", context
@@ -319,6 +319,9 @@ task integrationTests(type: Test) {
     systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") && !k.startsWith("RADAR_JDBC") }
     systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") && !k.startsWith("CDA_JDBC") }
     systemProperties += project.properties.findAll { k, v -> k.startsWith("testcontainer") }
+
+    environment("TS_USE_DMZ_INCLUDE", "true")
+
     jvmArgs += "-DwarFile=$buildDir/libs/${project.name}-${project.version}.war"
     jvmArgs += "-DwarContext=/cwms-data"
     jvmArgs += "-Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager"

cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java

@@ -1,6 +1,7 @@
 package cwms.cda.api;
 
 import static com.codahale.metrics.MetricRegistry.name;
+import static cwms.cda.ApiServlet.CWMS_USERS_ROLE;
 import static cwms.cda.api.Controllers.*;
 
 import com.codahale.metrics.Histogram;
@@ -29,6 +30,9 @@
 import cwms.cda.data.dto.csv.TimeSeriesCsv;
 import cwms.cda.formatters.Formats;
 import cwms.cda.helpers.DateUtils;
+import cwms.cda.security.CdaAccessManager;
+import cwms.cda.security.CwmsAuthException;
+import cwms.cda.security.Role;
 import io.javalin.apibuilder.CrudHandler;
 import io.javalin.core.util.Header;
 import io.javalin.core.validation.JavalinValidation;
@@ -41,6 +45,9 @@
 import io.javalin.plugin.openapi.annotations.OpenApiParam;
 import io.javalin.plugin.openapi.annotations.OpenApiRequestBody;
 import io.javalin.plugin.openapi.annotations.OpenApiResponse;
+import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID;
+import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN;
+
 import java.io.IOException;
 import java.io.StringWriter;
 import java.net.URISyntaxException;
@@ -52,11 +59,13 @@
 import java.util.concurrent.TimeUnit;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpStatus;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jooq.DSLContext;
 import org.jooq.exception.DataAccessException;
+import org.jooq.impl.DSL;
 
 public class TimeSeriesController implements CrudHandler {
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
@@ -121,6 +130,8 @@ public class TimeSeriesController implements CrudHandler {
     private final Histogram requestResultSize;
     static final int DEFAULT_PAGE_SIZE = 500;
 
+    private static final boolean USE_DMZ_LIST = Boolean.parseBoolean(System.getenv("TS_USE_DMZ_INCLUDE"));
+
 
     public TimeSeriesController(MetricRegistry metrics) {
         this.metrics = metrics;
@@ -445,7 +456,6 @@ public void getAll(@NotNull Context ctx) {
             TimeSeriesDao dao = getTimeSeriesDao(dsl);
             String format = ctx.queryParamAsClass(FORMAT, String.class).getOrDefault("");
             String names = requiredParam(ctx, NAME);
-
             //try 'unit' if 'units' is not provided as that was the original parameter name
             String units = ctx.queryParamAsClass(UNITS, String.class)
                     .getOrDefault(ctx.queryParamAsClass(UNIT, String.class)
@@ -498,7 +508,7 @@ public void getAll(@NotNull Context ctx) {
             ZonedDateTime endZdt = end != null
                     ? DateUtils.parseUserDate(end, timezone)
                     : ZonedDateTime.now(tz);
-
+            checkAuth(ctx, names, ctx.queryParam(OFFICE));
             if (version != null && version.equals("2")) {
 
                 String office = requiredParam(ctx, OFFICE);
@@ -620,6 +630,59 @@ public void getAll(@NotNull Context ctx) {
         }
     }
 
+    /**
+     * Presumably a stop gap until formal authorization is in place. Compares provided time series id to DMZ include TS
+     * Group and if the session is authorized to determine if the request should proceed.
+     * @param ctx
+     * @param dsl
+     * @param names
+     * @param office
+     */
+    public static void checkAuth(Context ctx, String names, String office)
+    {
+        if (!USE_DMZ_LIST)
+        {
+                return;
+        }
+        final var authEx = new CwmsAuthException("Required time series is not authorized for public release at this time.", HttpStatus.SC_UNAUTHORIZED);
+        var p = CdaAccessManager.getApiPrincipal(ctx);
+        if (p != null && p.getRoles().contains(new Role(CWMS_USERS_ROLE)))
+        {
+            logger.atInfo().log("access allowed, authenticated. %s", p.getName());
+            return; // at this time there are no access restrictions on "CWMS Users"
+        }
+        final var TS_ID = AV_CWMS_TS_ID.AV_CWMS_TS_ID;
+        final var TS_GROUP = AV_TS_GRP_ASSGN.AV_TS_GRP_ASSGN;
+        var dsl = JooqDao.getDslContext(ctx, null);
+        var countQuery =
+                dsl.selectOne()
+                .from(TS_GROUP)
+                .where(
+                        TS_GROUP.GROUP_ID.eq("DMZ Include List")
+                        .and(TS_GROUP.TS_CODE.in(
+                                dsl.select(TS_ID.TS_CODE)
+                                .from(TS_ID)
+                                // There are a few endpoints which may not required the office id
+                                // with a time series id. in that case office will be null, possibly
+                                // restricting data when it should not be. Unfortunately, as their may
+                                // be duplicate names, there is not a better way to handle that beside
+                                // requiring the office id, which is the behavior of any new endpoint.
+                                .where(TS_ID.DB_OFFICE_ID.equalIgnoreCase(office).and(
+                                       TS_ID.CWMS_TS_ID.equalIgnoreCase(names)))
+                                )
+                        )
+                );
+        boolean found = dsl.fetchExists(countQuery);
+
+        if (found)
+        {
+                logger.atInfo().log("access allowed, include list.");
+                return;
+        }
+        logger.atInfo().log("access denied, include list.");
+        throw authEx;
+    }
+
     private void streamCsv(@NotNull Context ctx, CsvConfiguration csvConfig, int batchSize, TimeSeriesDao dao, TimeSeriesRequestParameters requestParameters) {
         int csvBatchSize = validateCsvBatchSize(batchSize);
         dao.streamRequestedTimeSeriesCsv(

cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java

@@ -217,7 +217,7 @@ public void handle(@NotNull Context ctx) {
                     : ZonedDateTime.now(tz);
 
             String office = ctx.queryParam(OFFICE);
-
+            TimeSeriesController.checkAuth(ctx, names, office);
             FilteredTimeSeriesParameters ftsParams = FilteredTimeSeriesParameters.Builder.from(ctx)
                     .build();
 

cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java

@@ -49,6 +49,7 @@
 import cwms.cda.data.dto.RecentValue;
 import cwms.cda.formatters.ContentType;
 import cwms.cda.formatters.Formats;
+import cwms.cda.security.CwmsAuthException;
 import io.javalin.core.util.Header;
 import io.javalin.http.Context;
 import io.javalin.http.Handler;
@@ -125,7 +126,9 @@ private TimeSeriesDao getTimeSeriesDao(DSLContext dsl) {
                         + "implemented")
             },
             path = "/timeseries/recent",
-            description = "Returns CWMS Timeseries Groups Data",
+            description = "Returns CWMS Timeseries Groups Data. NOTE: as groups may be defined with restricted time series"
+                        + "for internal use, instead of the whole request returning a 403 error the restricted data is"
+                        + "instead silently removed so that rest of the group may be retrieved.",
             tags = TimeSeriesController.TAG,
             method = HttpMethod.GET
     )
@@ -181,6 +184,21 @@ public void handle(@NotNull Context ctx) {
             } else {
                 latestValues = dao.findMostRecentsInRange(office, tsIds, pastLimit, futureLimit, unitSystem);
             }
+            latestValues = latestValues.stream()
+                                       .filter(rv -> {
+                                           // while not the most performant code,
+                                           // it'll work. Additionally for users with authorization
+                                            // checkAuth bypasses any database calls
+                                           try {
+                                               TimeSeriesController.checkAuth(ctx,
+                                                                              rv.getDqu().getCwmsTsId(),
+                                                                              rv.getDqu().getOfficeId());
+                                               return true;
+                                           } catch (CwmsAuthException ex) {
+                                               return false;
+                                           }
+                                       })
+                                       .collect(Collectors.toList());
 
             String formatHeader = ctx.header(Header.ACCEPT);
             ContentType contentType = Formats.parseHeader(formatHeader, RecentValue.class);

cwms-data-api/src/main/java/cwms/cda/security/CdaAccessManager.java

@@ -68,7 +68,7 @@ private void checkRateLimit(Context ctx) {
         }
     }
 
-    private DataApiPrincipal getApiPrincipal(Context ctx) {
+    public static DataApiPrincipal getApiPrincipal(Context ctx) {
         return ctx.sessionAttribute(IdentityProvider.PRINCIPAL_KEY);
     }
 

cwms-data-api/src/test/java/cwms/cda/api/AccessManagerTestIT.java

@@ -5,6 +5,7 @@
 import fixtures.TestAccounts;
 import fixtures.users.UserSpecSource;
 import fixtures.users.annotation.AuthType;
+import fixtures.users.annotation.AuthType.UserType;
 import io.restassured.filter.log.LogDetail;
 import io.restassured.specification.RequestSpecification;
 
@@ -60,6 +61,58 @@ void cant_create_without_user(String authType, TestAccounts.KeyUser user, Reques
 			.statusCode(is(HttpServletResponse.SC_UNAUTHORIZED));
     }
 
+    @ParameterizedTest
+    @ArgumentsSource (UserSpecSource.class)
+    @AuthType(userTypes = {UserType.GUEST_AND_PRIVS})
+    void test_ts_auth_not_in_dmz_list(String authType, TestAccounts.KeyUser user, RequestSpecification authSpec) throws Exception
+    {
+
+        final String tsName = "Restricted.Stage.Inst.1Hour.0.AuthTest";
+        final String office = user.getOperatingOffice() == null ? "SPK" : user.getOperatingOffice();
+        createLocation("Restricted", true, office);
+        createTimeseries(office, tsName, false);
+        given()
+			.log().ifValidationFails(LogDetail.ALL,true)
+			.spec(authSpec)
+			.contentType("application/json")
+			.queryParam("office", office)
+            .queryParam("name", tsName)
+		.when()
+			.get(  "/timeseries")
+		.then()
+			.assertThat()
+			.log().ifValidationFails(LogDetail.ALL,true)
+			.statusCode(is(
+                user == TestAccounts.KeyUser.GUEST ||
+                user == TestAccounts.KeyUser.NONE ||
+                user == TestAccounts.KeyUser.SPK_CAC_BUT_NOT_CWMS_USER
+                     ? HttpServletResponse.SC_UNAUTHORIZED : HttpServletResponse.SC_OK));
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource (UserSpecSource.class)
+    @AuthType(userTypes = {UserType.GUEST_AND_PRIVS})
+    void test_ts_auth_in_dmz_list(String authType, TestAccounts.KeyUser user, RequestSpecification authSpec) throws Exception
+    {
+
+        final String tsName = "NotRestricted.Stage.Inst.1Hour.0.AuthTest";
+        final String office = user.getOperatingOffice() == null ? "SPK" : user.getOperatingOffice();
+        createLocation("NotRestricted", true, office);
+        createTimeseries(office, tsName, true);
+        given()
+			.log().ifValidationFails(LogDetail.ALL,true)
+			.spec(authSpec)
+			.contentType("application/json")
+			.queryParam("office", office)
+            .queryParam("name", tsName)
+		.when()
+			.get(  "/timeseries")
+		.then()
+			.assertThat()
+			.log().ifValidationFails(LogDetail.ALL,true)
+			.statusCode(is(HttpServletResponse.SC_OK));
+    }
+
     @ParameterizedTest
     @ArgumentsSource(UserSpecSource.class)
     @AuthType(userTypes = { AuthType.UserType.PRIVS })

cwms-data-api/src/test/java/cwms/cda/api/DataApiTestIT.java

@@ -490,6 +490,19 @@ protected static void deleteLocation(String location, String office) throws SQLE
      * @throws SQLException
      */
     protected static void createTimeseries(String office, String timeseries) throws SQLException {
+        createTimeseries(office, timeseries, true);
+    }
+
+    /**
+     * Create a timeseries (location must already exist), no data or other meta data will be set.
+     * This only creates the timeseries name. Not data or other parameters are set.
+     *
+     * @param office     owning office
+     * @param timeseries timeseries name
+     * @param inDmzList if this timeseries id should go in the DMZ include list.
+     * @throws SQLException
+     */
+    protected static void createTimeseries(String office, String timeseries, boolean inDmzList) throws SQLException {
         CwmsDatabaseContainer<?> db = CwmsDataApiSetupCallback.getDatabaseLink();
         db.connection((c) -> {
             try (PreparedStatement stmt = c.prepareStatement(createTimeseriesQuery)) {
@@ -503,6 +516,36 @@ protected static void createTimeseries(String office, String timeseries) throws
                 throw new RuntimeException("Unable to create timeseries", ex);
             }
         }, "cwms_20");
+        if (inDmzList) {
+            assignTsToDmzInclude(office, timeseries);
+        }
+    }
+
+    /**
+     * Stop gap to make establishing timeseries for tests easier
+     * without disabling the time series auth check.
+     * NOTE: this method currently assumes the database default is still using ~ for local regular
+     * instead of the new intervals with Local in the name. It will eventually need to be adjusted.
+     * Though preferably as this is part of a stop gap change, this method would just go away.
+     * @param office
+     * @param timeSeries
+     * @throws SQLException
+     */
+    protected static void assignTsToDmzInclude(String office, String timeSeries) throws SQLException {
+        CwmsDatabaseContainer<?> db = CwmsDataApiSetupCallback.getDatabaseLink();
+        final String query = "begin cwms_ts.assign_ts_group(p_ts_category_id => 'Data Dissemination', p_ts_group_id => 'DMZ Include List', p_ts_id => ?, p_db_office_id => ?); end;";
+        String ts = timeSeries.replaceFirst("(\\d+.*?)Local", "~$1");
+        logger.atInfo().log("Replacing %s with %s", timeSeries, ts);
+        db.connection((c) -> {
+            try (PreparedStatement stmt = c.prepareStatement(query)) {
+                // at this time the default is using non-local names
+                stmt.setString(1, ts);
+                stmt.setString(2, office);
+                stmt.execute();
+            } catch (SQLException ex) {
+                throw new RuntimeException("Unable to create timeseries", ex);
+            }
+        }, "cwms_20");
     }
 
 
@@ -521,6 +564,7 @@ protected static void createTimeseries(String office, String timeseries, int off
                 throw new RuntimeException("Unable to create timeseries", ex);
             }
         }, "cwms_20");
+        assignTsToDmzInclude(office, timeseries);
     }
 
     protected static void createTimeseriesWithNewLRTSInterval(String office, String timeseries, int offset) throws SQLException {
@@ -541,6 +585,7 @@ protected static void createTimeseriesWithNewLRTSInterval(String office, String
                 throw new RuntimeException("Unable to create timeseries", ex);
             }
         }, "cwms_20");
+        assignTsToDmzInclude(office, timeseries);
     }
 
     /**