Fs 3356 mark as complete (#228)

Note - commit history looks wonky as I accidentally pushed to main initially. Then reverted on main, but had to revert the reverting in my branch to reinstate the changes.

* Revert "Revert "fs-3356 integrated with form-builder""

This reverts commit 2ec0bef.

* Revert "Revert "Merge branch 'main' of https://github.com/communitiesuk/funding-service-design-application-store""

This reverts commit 3c501cf.

.env.development

@@ -2,6 +2,8 @@ FLASK_APP=app.py
 FLASK_ENV=development
 FLASK_RUN_PORT=5000
 FLASK_RUN_HOST=localhost
+
+# Below credentials belongs to local development server. Do not commit any external services credentials
 DATABASE_URL=postgresql://postgres:password@localhost:5432/application_store
 FUND_STORE_API_HOST=http://localhost:3001
 AWS_ACCESS_KEY_ID=FSDIOSFODNN7EXAMPLE

.github/workflows/manual-dev-deploy.yaml

@@ -48,10 +48,10 @@
         perf_test_target_url_application_store: https://funding-service-design-application-store-dev.london.cloudapps.digital
         perf_test_target_url_fund_store: https://funding-service-design-fund-store-dev.london.cloudapps.digital
         perf_test_target_url_assessment_store: https://funding-service-design-assessment-store-dev.london.cloudapps.digital
-        e2e_tests_target_url_frontend: https://fsd:fsd@frontend.dev.gids.dev
-        e2e_tests_target_url_authenticator: https://fsd:fsd@authenticator2.dev.gids.dev
-        e2e_tests_target_url_form_runner: https://fsd:fsd@forms.dev.gids.dev
-        e2e_tests_target_url_assessment: https://fsd:fsd@assessment.dev.gids.dev
+        e2e_tests_target_url_frontend: https://fsd:fsd@frontend.dev.gids.dev  # pragma: allowlist secret
+        e2e_tests_target_url_authenticator: https://fsd:fsd@authenticator2.dev.gids.dev  # pragma: allowlist secret
+        e2e_tests_target_url_form_runner: https://fsd:fsd@forms.dev.gids.dev  # pragma: allowlist secret
+        e2e_tests_target_url_assessment: https://fsd:fsd@assessment.dev.gids.dev  # pragma: allowlist secret
         run_performance_tests: true
         run_e2e_tests: false
       secrets:

.pre-commit-config.yaml

@@ -29,3 +29,9 @@ repos:
     hooks:
     -   id: pyupgrade
         args: ["--py39-plus"]
+- repo: https://github.com/Yelp/detect-secrets
+  rev: v1.4.0
+  hooks:
+  -   id: detect-secrets
+      args: ['--disable-plugin', 'HexHighEntropyString']
+      exclude: .env.development

README.md

@@ -58,7 +58,10 @@ Set the environment variables "DATABASE_URL" to your postgres connection string
 
 Eg.
 
-`export DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres`
+```
+# pragma: allowlist nextline secret
+export DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres
+```
 
 
 ### Build with Paketo
@@ -204,7 +207,7 @@ To seed applicaitons, we need the completed form json. If you have that, skip to
 1. Get a submitted application into your local DB. You can either do this manually or by running the automated tests against your local docker runner.
 1. Find the `application_id` of that submitted application._
 1. Edit the [tests file](/tests/test_seed_db.py) to un-skip `test_retrieve_test_data` and then set `target_app` to be the `application_id` you just submitted.
-1. Update your unit test config to point at the same DB as the docker runner. Update [pytest.ini](/pytest.ini) so that `D:DATABASE_URL` points at the docker runner application store db: `D:DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5433/application_store`
+1. Update your unit test config to point at the same DB as the docker runner. Update [pytest.ini](/pytest.ini) so that `D:DATABASE_URL` points at the docker runner application store db: `D:DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5433/application_store  # pragma: allowlist secret`
 1. Run the single test `test_retrieve_test_data` - this should output the json of all the completed forms for that application into funding-service-design-store/forms.json.
 1. Copy this file into [seed_data](/tests/seed_data/) and name it `<fund_short_code>_<round_short_code>_all_forms.json`.
 1. *IMPORTANT* Change the config in [pytest.ini](/pytest.ini) back to what it was so you don't accidentally wipe your docker runner DB next time you run tests!

config/envs/unit_testing.py

@@ -10,7 +10,7 @@
 @configclass
 class UnitTestingConfig(DefaultConfig):
     #  Application Config
-    SECRET_KEY = "dev"
+    SECRET_KEY = "dev"  # pragma: allowlist secret
     SESSION_COOKIE_NAME = CommonConfig.SESSION_COOKIE_NAME
     FLASK_ENV = "unit_test"
     FUND_STORE_API_HOST = DefaultConfig.TEST_FUND_STORE_API_HOST
@@ -21,7 +21,7 @@ class UnitTestingConfig(DefaultConfig):
     # Database
     SQLALCHEMY_DATABASE_URI = environ.get(
         "DATABASE_URL",
-        "postgresql://postgres:postgres@localhost:5432/fsd_app_store_test",
+        "postgresql://postgres:postgres@localhost:5432/fsd_app_store_test",  # pragma: allowlist secret
     )
     SQLALCHEMY_TRACK_MODIFICATIONS = False
 

copilot/environments/addons/funding-service-magic-links.yml

@@ -90,7 +90,7 @@ Outputs:
     Value:
       !Sub
       - "rediss://:${PASSWORD}@${HOSTNAME}:${PORT}"
-      - PASSWORD: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref 'RedisSecret', ":SecretString:password}}" ]]
+      - PASSWORD: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref 'RedisSecret', ":SecretString:password}}" ]]  # pragma: allowlist secret
         HOSTNAME: !GetAtt 'Redis.RedisEndpoint.Address'
         PORT: !GetAtt 'Redis.RedisEndpoint.Port'
     Export:

copilot/fsd-application-store/addons/fsd-application-store-cluster.yml

@@ -85,9 +85,9 @@ Resources:
     Type: 'AWS::RDS::DBCluster'
     Properties:
       MasterUsername:
-        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:username}}" ]]
+        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:username}}" ]]  # pragma: allowlist secret
       MasterUserPassword:
-        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:password}}" ]]
+        !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:password}}" ]]  # pragma: allowlist secret
       DatabaseName: !Ref fsdapplicationstoreclusterDBName
       Engine: 'aurora-postgresql'
       EngineVersion: '14.4'
@@ -126,11 +126,11 @@ Outputs:
     Value:
       !Sub
       - "postgres://${USERNAME}:${PASSWORD}@${HOSTNAME}:${PORT}/${DBNAME}"
-      - USERNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:username}}" ]]
-        PASSWORD: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:password}}" ]]
-        HOSTNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:host}}" ]]
-        PORT: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:port}}" ]]
-        DBNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:dbname}}" ]]
+      - USERNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:username}}" ]]  # pragma: allowlist secret
+        PASSWORD: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:password}}" ]]  # pragma: allowlist secret
+        HOSTNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:host}}" ]]  # pragma: allowlist secret
+        PORT: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:port}}" ]]  # pragma: allowlist secret
+        DBNAME: !Join [ "",  [ '{{resolve:secretsmanager:', !Ref fsdapplicationstoreclusterAuroraSecret, ":SecretString:dbname}}" ]]  # pragma: allowlist secret
 
   fsdapplicationstoreclusterSecret: # injected as FSDAPPLICATIONSTORECLUSTER_SECRET environment variable by Copilot.
     Description: "The JSON secret that holds the database username and password. Fields are 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' and 'engine'"

db/queries/statuses/queries.py

@@ -66,6 +66,7 @@ def update_application_status(
 
 def update_form_status(
     form_to_update: Forms,
+    round_mark_as_complete_enabled: bool,
     is_summary_page_submitted: bool = False,
 ):
     """
@@ -78,14 +79,46 @@ def update_form_status(
 
     """
 
-    status_list = [question_page["status"] for question_page in form_to_update.json]
+    if round_mark_as_complete_enabled:
+        mark_as_complete_question = next(
+            (
+                question_page
+                for question_page in form_to_update.json
+                if question_page["question"] == "MarkAsComplete"
+            ),
+            None,
+        )
+        is_marked_as_complete = (
+            mark_as_complete_question["fields"][0]["answer"]
+            if mark_as_complete_question
+            else False
+        )
+    else:
+        is_marked_as_complete = False
+
+    status_list = [
+        question_page["status"]
+        for question_page in form_to_update.json
+        if question_page["question"] != "MarkAsComplete"
+    ]
     if "COMPLETED" not in status_list:
+        # If no single question page is complete
         form_to_update.status = "NOT_STARTED"
-    elif "NOT_STARTED" not in status_list and form_to_update.has_completed:
-        form_to_update.status = "COMPLETED"
     elif "NOT_STARTED" not in status_list and is_summary_page_submitted:
+        # If every question page has answers and this is submit on summary page
+        if round_mark_as_complete_enabled:
+            if is_marked_as_complete:
+                form_to_update.status = "COMPLETED"
+                form_to_update.has_completed = True
+            else:
+                form_to_update.status = "IN_PROGRESS"
+                form_to_update.has_completed = False
+        else:
+            form_to_update.status = "COMPLETED"
+            form_to_update.has_completed = True
+    elif "NOT_STARTED" not in status_list and form_to_update.has_completed:
+        # All question pages have answers and form has previously completed
         form_to_update.status = "COMPLETED"
-        form_to_update.has_completed = True
     else:
         form_to_update.status = "IN_PROGRESS"
 
@@ -188,13 +221,15 @@ def update_statuses(
         form_name (`str`): Name of the form that has been updated, uses this form as the basis for the update
         is_summary_page_submitted (`bool`): If this is as a result of submitting from the summary page of a form.
     """
+    application = get_application(application_id, include_forms=True)
+    round = get_round(application.fund_id, application.round_id)
     if form_name:
         form_to_update = get_form(application_id=application_id, form_name=form_name)
         update_question_page_statuses(stored_form_json=form_to_update.json)
-        update_form_status(form_to_update, is_summary_page_submitted)
+        update_form_status(
+            form_to_update, round.mark_as_complete_enabled, is_summary_page_submitted
+        )
         db.session.commit()
 
-    application = get_application(application_id, include_forms=True)
-    round = get_round(application.fund_id, application.round_id)
     update_application_status(application, round.requires_feedback)
     db.session.commit()

external_services/models/round.py

@@ -14,6 +14,7 @@ class Round:
     contact_email: str
     requires_feedback: bool = False
     project_name_field_id: Optional[str] = None
+    mark_as_complete_enabled: bool = False
 
     @staticmethod
     def from_json(data: dict):
@@ -28,4 +29,5 @@ def from_json(data: dict):
             project_name_field_id=data.get("project_name_field_id", None),
             contact_email=data.get("contact_email", None),
             requires_feedback=data.get("requires_feedback") or False,
+            mark_as_complete_enabled=data.get("mark_as_complete_enabled") or False,
         )

pytest.ini

@@ -2,6 +2,7 @@
 env =
     FLASK_ENV=unit_test
     FLASK_DEBUG=1
+    # pragma: allowlist nextline secret
     D:DATABASE_URL=postgresql://postgres:postgres@127.0.0.1:5432/fsd_app_store_test
     GITHUB_SHA=abc123
 mocked-sessions=db.db.session

tests/README.md

@@ -1,7 +1,6 @@
 # How to run tests
 
-1. Ensure that the environment variable DATABASE_URL is set to your db's connection string. The default during tests is:
-   `postgresql://postgres:postgres@127.0.0.1:5432/fsd_app_store_test`, unless you manually set DATABASE_URL.
+1. Ensure that the environment variable DATABASE_URL is set to your db's connection string.
    - The command `invoke bootstrap-test-db --database-host=your-db-url` will create a db called "fsd_app_store_test".
 2. Ensure you have installed the requirements-dev.txt
 3. Run `pytest`

tests/test_application_status.py

@@ -81,48 +81,147 @@ def test_update_question_statuses(form_json, exp_status):
 
 
 @pytest.mark.parametrize(
-    "form_json,form_has_completed,is_summary_submit,exp_status,exp_has_completed",
+    "form_json,form_has_completed,is_summary_submit,round_mark_as_complete_enabled,"
+    " mark_as_complete, exp_status,exp_has_completed",
     [
-        ([{"status": "NOT_STARTED"}], False, False, "NOT_STARTED", False),
+        (  # Previously marked as complete, want to mark as not complete
+            [
+                {"status": "COMPLETED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            True,
+            True,
+            True,
+            False,
+            "IN_PROGRESS",
+            False,
+        ),
+        (  # Marking as complete for the first time
+            [
+                {"status": "COMPLETED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            False,
+            True,
+            True,
+            True,
+            "COMPLETED",
+            True,
+        ),
+        (  # Not on summary page, not marking as complete
+            [
+                {"status": "COMPLETED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            False,
+            True,
+            True,
+            False,
+            "IN_PROGRESS",
+            False,
+        ),
+        (
+            [{"status": "NOT_STARTED", "question": "abc"}],
+            False,
+            False,
+            False,
+            None,
+            "NOT_STARTED",
+            False,
+        ),
         (
-            [{"status": "IN_PROGRESS"}, {"status": "COMPLETED"}],
+            [
+                {"status": "IN_PROGRESS", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            False,
             False,
             False,
+            None,
             "IN_PROGRESS",
             False,
         ),
         (
-            [{"status": "NOT_STARTED"}, {"status": "COMPLETED"}],
+            [
+                {"status": "NOT_STARTED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
             False,
             False,
+            False,
+            None,
             "IN_PROGRESS",
             False,
         ),
         (
-            [{"status": "COMPLETED"}, {"status": "COMPLETED"}],
+            [
+                {"status": "COMPLETED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            False,
             False,
             False,
+            None,
             "IN_PROGRESS",
             False,
         ),
         (
-            [{"status": "COMPLETED"}, {"status": "COMPLETED"}],
+            [
+                {"status": "COMPLETED", "question": "abc"},
+                {"status": "COMPLETED", "question": "abc"},
+            ],
+            False,
+            True,
+            False,
+            None,
+            "COMPLETED",
+            True,
+        ),
+        (
+            [{"status": "NOT_STARTED", "question": "abc"}],
+            True,
+            False,
             False,
+            None,
+            "NOT_STARTED",
             True,
+        ),
+        (
+            [{"status": "COMPLETED", "question": "abc"}],
+            True,
+            False,
+            False,
+            None,
             "COMPLETED",
             True,
         ),
-        ([{"status": "NOT_STARTED"}], True, False, "NOT_STARTED", True),
-        ([{"status": "COMPLETED"}], True, False, "COMPLETED", True),
     ],
 )
 def test_update_form_status(
-    form_json, form_has_completed, is_summary_submit, exp_status, exp_has_completed
+    form_json,
+    form_has_completed,
+    is_summary_submit,
+    round_mark_as_complete_enabled,
+    mark_as_complete,
+    exp_status,
+    exp_has_completed,
 ):
     form_to_update = MagicMock()
     form_to_update.json = form_json
     form_to_update.has_completed = form_has_completed
-    update_form_status(form_to_update, is_summary_submit)
+
+    # If a round doesn't use mark_as_complete, the question is not in the json
+    if mark_as_complete is not None:
+        form_to_update.json.append(
+            {
+                "status": "COMPLETED",
+                "question": "MarkAsComplete",
+                "fields": [{"answer": mark_as_complete}],
+            },
+        )
+    update_form_status(
+        form_to_update, round_mark_as_complete_enabled, is_summary_submit
+    )
     assert form_to_update.status == exp_status
     assert form_to_update.has_completed == exp_has_completed