docs: add udfs to docs

docs/advanced_guidance/package_documentation/auditing.md

@@ -11,3 +11,4 @@
             - get_all_business_rule_submissions
             - get_all_error_report_submissions
             - get_current_processing_info
+        show_root_toc_entry: false

docs/advanced_guidance/package_documentation/business_rules/index.md

@@ -0,0 +1,31 @@
+---
+title: Business Rules Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Operations__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Operation` objects here](operations.md)
+
+-   :material-language-python:{ .lg .middle } __Reference Data Loaders__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Business Rules` objects here](refdata_loaders.md)
+
+-   :material-language-python:{ .lg .middle } __Reference Data Types__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Data Contract` objects here](reference_data_types.md)
+
+-   :material-language-python:{ .lg .middle } __User Defined Functions (UDFs)__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `UDFs` objects here](udfs.md)
+
+</div>

docs/advanced_guidance/package_documentation/business_rules/udfs.md

@@ -0,0 +1,10 @@
+
+!!! warning
+
+    Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets.
+
+::: dve.core_engine.functions.implementations
+    handler: python
+    options:
+        show_root_heading: true
+        heading_level: 1

docs/advanced_guidance/package_documentation/data_contract/index.md

@@ -0,0 +1,20 @@
+---
+title: Data Contract Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Readers__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Readers` objects here](readers.md)
+
+-   :material-language-python:{ .lg .middle } __Domain Types__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Domain Types` objects here](domain_types.md)
+
+
+</div>

docs/advanced_guidance/package_documentation/entity_hierarchy.md

@@ -2,4 +2,4 @@
     handler: python
     options:
         show_root_heading: true
-        heading_level: 2
+        heading_level: 1

docs/advanced_guidance/package_documentation/feedback/feedback_messages.md

@@ -0,0 +1,9 @@
+
+::: dve.core_engine.message
+    handler: python
+    options:
+        heading_level: 2
+        members:
+            - FeedbackMessage
+            - UserMessage
+        show_root_toc_entry: false

docs/advanced_guidance/package_documentation/feedback/index.md

@@ -0,0 +1,13 @@
+---
+title: User Feedback Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Feedback Messages__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Operation` objects here](feedback_messages.md)
+
+</div>

docs/advanced_guidance/package_documentation/index.md

@@ -4,16 +4,46 @@ title: Package Documentation
 
 <div class="grid cards" markdown>
 
--   :material-language-python:{ .lg .middle } __Pipeline__
+-   :material-database-plus:{ .lg .middle } __Auditing__
 
     ---
 
-    [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
+    [:octicons-arrow-right-24: Read about the `Auditing` objects here](auditing.md)
+
+-   :material-clipboard-check:{ .lg .middle } __Business Rules__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Business Rules` objects here](./business_rules/index.md)
+
+-   :material-file-document-check:{ .lg .middle } __Data Contract__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Data Contract` objects here](./data_contract/index.md)
+
+-   :material-relation-one-to-many:{ .lg .middle } __Entity Relationships__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Entity Reliationships` objects here](entity_hierarchy.md)
+
+-   :material-message-alert:{ .lg .middle } __Submission Feedback__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](./feedback/index.md)
 
--   :material-set-all:{ .lg .middle } __Reference Data Loader__
+-   :material-package-variant-plus:{ .lg .middle } __Additional Objects__
 
     ---
 
-    [:octicons-arrow-right-24: Read about the `Reference Data Loader` objects here](refdata_loaders.md)
+    [:octicons-arrow-right-24: Read about the additional objects here](./other/index.md)
+
+-   :material-pipe:{ .lg .middle } __Pipeline__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
 
 </div>

docs/advanced_guidance/package_documentation/other/index.md

@@ -0,0 +1,13 @@
+---
+title: Additional Objects Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-code-braces-box:{ .lg .middle } __Models__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the Model objects here](./models.md)
+
+</div>

docs/advanced_guidance/package_documentation/models.md → docs/advanced_guidance/package_documentation/other/models.md

@@ -2,5 +2,7 @@
 ::: dve.core_engine.models
     handler: python
     options:
-        show_root_heading: true
         heading_level: 2
+        members:
+            - SubmissionInfo
+        show_root_toc_entry: false

docs/user_guidance/business_rules.md

@@ -209,7 +209,27 @@ The difference between modifiying the existing entity and adding a new one is si
 
 ### Operations
 
-For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/operations.md).
+For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/business_rules/operations.md).
+
+### User Defined Functions (UDFs)
+
+!!! warning
+
+    Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets.
+
+
+The DVE currently has a number of UDFs available to be called during a rule check. For instance, if you wanted to use the [NHS Number](https://www.datadictionary.nhs.uk/attributes/nhs_number.html#ariaid-title2) mod 11 check then you can simply call the udf during the expression definition - such as:
+
+```json title="example.dischema.json"
+    {
+        "entity": "example",
+        "name": "check NHS number passes mod 11 check",
+        "expression": "nhsno_mod11_check(<name_of_field>)"
+        ...
+    }
+```
+
+A full list of UDF functions available can be found [here](../advanced_guidance/package_documentation/business_rules/udfs.md).
 
 ## Post Rule
 
@@ -358,7 +378,7 @@ If your Business Rules are reliant on reference data, then you can add the `"ref
 
         - Refdata entities are also immutable. So, if you need to modify them in any way, you will always need to create a new entity from it
 
-For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/refence_data_types.md).
+For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/business_rules/reference_data_types.md).
 
 ## Complex Rules
 

docs/user_guidance/data_contract.md

@@ -278,7 +278,7 @@ As you can see, we can set the "type" for several fields to `MyConstrainedString
 
 #### Domain Types
 
-Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/domain_types.md).
+Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/data_contract/domain_types.md).
 
 ### Complex Types
 

docs/user_guidance/feedback_messages.md

@@ -28,4 +28,4 @@ If the DVE is crashing and the error message is either unreadable or is crashing
 
 ## Feedback Message Models
 
-Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback_messages.md).
+Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback/feedback_messages.md).

docs/user_guidance/file_transformation.md

@@ -155,7 +155,7 @@ The secondary use of the File Transformation stage is the ability to normalise y
     ```
 
 !!! abstract ""
-    You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/readers.md).
+    You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/data_contract/readers.md).
 
 ## Supported Formats
 

docs/user_guidance/implementations/duckdb.md

@@ -36,7 +36,7 @@ Now you have the DuckDB connection object setup, you are ready to setup the requ
 
 ## Generating SubmissionInfo objects
 
-Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path:
+Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/other/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path:
 ```py
 import glob
 from datetime import date, datetime
@@ -112,7 +112,7 @@ DuckDBRefDataLoader.dataset_config_uri = Path("path", "to", "my", "rules").as_po
 
 The connection passed into the `DuckDBRefDataLoader` object will then be able to use various DuckDB readers to load data from an existing table on the connection OR loading data from reference data persisted in either `parquet` or `pyarrow` format.
 
-If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/refdata_loaders.md).
+If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md).
 
 Now we can move onto setting up the DuckDB DVE Pipeline object.
 
@@ -171,4 +171,4 @@ error_reports = dve_pipeline.cluster_pipeline_run()
 For further details on the objects referenced above, you can use the following links to read more about the objects:
 
 - [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md)
-- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md)
+- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md)

docs/user_guidance/implementations/spark.md

@@ -187,4 +187,4 @@ error_reports = dve_pipeline.cluster_pipeline_run()
 For further details on the objects referenced above, you can use the following links to read more about the objects:
 
 - [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md)
-- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md)
+- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md)

src/dve/core_engine/functions/implementations.py

@@ -42,116 +42,6 @@ def date_in_current_financial_year(test_date: datetime.date) -> bool:
     return test_date >= current_fiscal_year_start
 
 
-def is_valid_ods_code(check_ods_code: str) -> bool:
-    """A test ODS code lookup, contains some actual valid codes"""
-    if (not check_ods_code) or (not isinstance(check_ods_code, str)):
-        return False
-    valid_ods_codes = [
-        "EE142976",
-        "EE144430",
-        "EE143473",
-        "EE148112",
-        "EE142863",
-        "EE147862",
-        "EE142472",
-        "EE141208",
-        "EE143149",
-        "EE140862",
-        "EE140319",
-        "EE144899",
-        "EE144475",
-        "EE141850",
-        "EE147934",
-        "EE141068",
-        "EE143825",
-        "EE147805",
-        "EE143489",
-        "EE146813",
-        "EE145703",
-        "EE148295",
-        "EE140156",
-        "EE145502",
-        "EE148396",
-        "EE144126",
-        "EE145590",
-        "EE141566",
-        "EE142081",
-        "EE143640",
-        "EE144911",
-        "EE145935",
-        "EE145279",
-        "EE143156",
-        "EE146556",
-        "EE140781",
-        "EE144734",
-        "EE144841",
-        "EE140419",
-        "EE140040",
-        "EE147342",
-        "EE143330",
-        "EE140926",
-        "EE140926",
-        "EE146438",
-        "EE142137",
-        "EE143856",
-        "EE141067",
-        "EE148534",
-        "EE141310",
-        "EE146899",
-        "EE146996",
-        "EE147487",
-        "EE148447",
-        "EE144311",
-        "EE142147",
-        "EE147605",
-        "EE142117",
-        "EE144087",
-        "EE147326",
-        "EE147614",
-        "EE143703",
-        "EE146135",
-        "EE140782",
-        "EE143603",
-        "EE143554",
-        "EE146659",
-        "EE140321",
-        "EE141185",
-        "EE147648",
-        "EE144527",
-        "EE142680",
-        "EE141620",
-        "EE145274",
-        "EE146251",
-        "EE148209",
-        "EE142574",
-        "EE148162",
-        "EE143118",
-        "EE142977",
-        "EE147798",
-        "EE147902",
-        "EE145780",
-        "EE146992",
-        "EE142916",
-        "EE144777",
-        "EE146935",
-        "EE145586",
-        "EE144570",
-        "EE147122",
-        "EE140874",
-        "EE141338",
-        "EE143244",
-    ]
-    return check_ods_code in valid_ods_codes
-
-
-def is_valid_national_org(check_org_code: str) -> bool:
-    """Simple test org code lookup"""
-    if (not check_org_code) or (not isinstance(check_org_code, str)):
-        return False
-    valid_org_codes = ["ORG01", "ORG02"]
-    return check_org_code in valid_org_codes
-
-
 def check_correct_numeric_signage(
     val: Union[float, int, Decimal], expected_sign: str = "+/-"
 ) -> Optional[bool]: