diff --git a/docs/builders.rst b/docs/builders.rst index 52e37fc43..d2a4c8778 100644 --- a/docs/builders.rst +++ b/docs/builders.rst @@ -163,3 +163,233 @@ or .. hint:: As an alternative, you can set the config option :ref:`needs_build_needumls` to export the needumls files during each build. + +.. _needs_per_page_builder: + +needs_per_page +-------------- +.. versionadded:: 1.4.0 + +The **needs_per_page** builder exports all found needs with same ``docname`` into separate ``json`` file. +If docname has slash like ``directives/list2need``, the file will be located in folder called :ref:`needs_per_page_build_path`. +e.g. `needs_per_page/directives/list2need.json` . + +Usage ++++++ + + +.. code-block:: bash + + sphinx-build -b needs_per_page source_dir build_dir + + +Format +++++++ +.. code-block:: python + + { + "needs": [ + { + "xyz_123": { + "docname": "configuration", + "doctype": ".rst", + "lineno": 203, + "target_id": "xyz_123", + "external_url": null, + "content_id": "xyz_123", + "type": "req", + "type_name": "Requirement", + "type_prefix": "R_", + "type_color": "#BFD8D2", + "type_style": "node", + "status": "open", + "tags": [], + "constraints": [], + "constraints_passed": null, + "constraints_results": {}, + "id": "xyz_123", + "title": "My requirement with custom options", + "full_title": "My requirement with custom options", + "content": "Some content", + "collapse": null, + "arch": {}, + "style": null, + "layout": "", + "template": null, + "pre_template": null, + "post_template": null, + "hide": false, + "delete": null, + "jinja_content": null, + "parts": {}, + "is_part": false, + "is_need": true, + "is_external": false, + "external_css": "external_link", + "is_modified": false, + "modifications": 0, + "my_extra_option": "A new option", + "another_option": "filter_me", + "author": "", + "comment": "", + "amount": "", + "hours": "", + "image": "", + "config": "", + "github": "", + "value": "", + "unit": "", + "query": "", + "specific": "", + "max_amount": "", + "max_content_lines": "", + "id_prefix": "", + "user": "", + "created_at": "", + "updated_at": "", + "closed_at": "", + "service": "", + "url": "", + "avatar": "", + "params": "", + "prefix": "", + "url_postfix": "", + "hidden": "", + "duration": "", + "completion": "", + "has_dead_links": "", + "has_forbidden_dead_links": "", + "links": [], + "links_back": [], + "parent_needs": [], + "parent_needs_back": [], + "blocks": [], + "blocks_back": [], + "tests": [], + "tests_back": [], + "checks": [], + "checks_back": [], + "triggers": [], + "triggers_back": [], + "starts_with": [], + "starts_with_back": [], + "starts_after": [], + "starts_after_back": [], + "ends_with": [], + "ends_with_back": [], + "sections": [ + "needs_extra_options", + "Options", + "Configuration" + ], + "section_name": "needs_extra_options", + "signature": "", + "parent_need": "", + "id_parent": "xyz_123", + "id_complete": "xyz_123" + } + }, + { + "EXTRA_REQ_001": { + "docname": "configuration", + "doctype": ".rst", + "lineno": 371, + "target_id": "EXTRA_REQ_001", + "external_url": null, + "content_id": "EXTRA_REQ_001", + "type": "req", + "type_name": "Requirement", + "type_prefix": "R_", + "type_color": "#BFD8D2", + "type_style": "node", + "status": null, + "tags": [], + "constraints": [], + "constraints_passed": null, + "constraints_results": {}, + "id": "EXTRA_REQ_001", + "title": "My requirement", + "full_title": "My requirement", + "content": "", + "collapse": null, + "arch": {}, + "style": null, + "layout": "", + "template": null, + "pre_template": null, + "post_template": null, + "hide": false, + "delete": null, + "jinja_content": null, + "parts": {}, + "is_part": false, + "is_need": true, + "is_external": false, + "external_css": "external_link", + "is_modified": false, + "modifications": 0, + "my_extra_option": "", + "another_option": "", + "author": "", + "comment": "", + "amount": "", + "hours": "", + "image": "", + "config": "", + "github": "", + "value": "", + "unit": "", + "query": "", + "specific": "", + "max_amount": "", + "max_content_lines": "", + "id_prefix": "", + "user": "", + "created_at": "", + "updated_at": "", + "closed_at": "", + "service": "", + "url": "", + "avatar": "", + "params": "", + "prefix": "", + "url_postfix": "", + "hidden": "", + "duration": "", + "completion": "", + "has_dead_links": "", + "has_forbidden_dead_links": "", + "links": [], + "links_back": [], + "parent_needs": [], + "parent_needs_back": [], + "blocks": [], + "blocks_back": [], + "tests": [], + "tests_back": [], + "checks": [], + "checks_back": [ + "EXTRA_TEST_001" + ], + "triggers": [], + "triggers_back": [], + "starts_with": [], + "starts_with_back": [], + "starts_after": [], + "starts_after_back": [], + "ends_with": [], + "ends_with_back": [], + "sections": [ + "needs_extra_links", + "Options", + "Configuration" + ], + "section_name": "needs_extra_links", + "signature": "", + "parent_need": "", + "id_parent": "EXTRA_REQ_001", + "id_complete": "EXTRA_REQ_001" + } + } + ] +} diff --git a/docs/changelog.rst b/docs/changelog.rst index 5a8b0ca72..6a89f6632 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -16,6 +16,8 @@ Released: under development ----- Released: under development +* Improvement: Added Builder :ref:`needs_per_page_builder:` added and config option :ref:`needs_per_page_build_path` + * Improvement: Reduce document build time, by memoizing the inline parse in ``build_need`` (`#968 `_) * Change `NeedsBuilder` format to `needs` (`#978 `_) diff --git a/docs/conf.py b/docs/conf.py index e6d47640e..3555be6d1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -361,6 +361,9 @@ def custom_defined_func(): # build needs.json to make permalinks work needs_build_json = True +# build json file include needs with the same docs_name +needs_per_page = True + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/configuration.rst b/docs/configuration.rst index 6cb9532ed..0afdcbc37 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -2289,3 +2289,34 @@ If true, need options like status, tags or links are collapsed and shown only af Default value: True Can be overwritten for each single need by setting :ref:`need_collapse`. + + +.. _needs_per_page: + +needs_per_page +~~~~~~~~~~~~~~ + +.. versionadded:: 1.4.0 + +Build list of ``json`` files that contain all found needs with the same ``docname``. The name of each file should match the ``docname``. + +This option works like :ref:`needs_build_json`. + +Default: False + + +.. _needs_per_page_build_path: + +needs_per_page_build_path +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 1.4.0 + +This option sets the location of list of ``json`` files that contain all found needs with the same ``docname``. + +Default value: need_per_page + + +.. hint:: + +The created ``need_per_page`` folder gets stored in the ``outdir`` of the current builder folder. This is e.g `_build/needs_per_page/directives/list2need.json` diff --git a/sphinx_needs/builder.py b/sphinx_needs/builder.py index 4053e6c92..eb2dd246f 100644 --- a/sphinx_needs/builder.py +++ b/sphinx_needs/builder.py @@ -1,3 +1,4 @@ +import json import os from typing import Iterable, Optional, Set @@ -6,6 +7,7 @@ from sphinx.application import Sphinx from sphinx.builders import Builder +from sphinx_needs.filter_common import filter_needs from sphinx_needs.logging import get_logger from sphinx_needs.needsfile import NeedsList from sphinx_needs.utils import unwrap @@ -43,9 +45,6 @@ def finish(self) -> None: # This is needed as needs could have been removed from documentation and if this is the case, # removed needs would stay in needs_list, if list gets not cleaned. needs_list.wipe_version(version) - # - from sphinx_needs.filter_common import filter_needs - filter_string = self.app.config.needs_builder_filter filtered_needs = filter_needs(self.app, needs, filter_string) @@ -157,3 +156,93 @@ def build_needumls_pumls(app: Sphinx, _exception: Exception) -> None: needs_builder.set_environment(env) needs_builder.finish() + + +class NeedsPerPageBuilder(Builder): + + """Json builder for needs, which creates separate docname-based json-files include all needs with same docname""" + + name = "needs_per_page" + format = "needs" + file_suffix = ".txt" + links_suffix = None + + def write_doc(self, docname: str, doctree: nodes.document) -> None: + pass + + def finish(self) -> None: + env = unwrap(self.env) + needs = env.needs_all_needs.values() + needs_list = NeedsList(env.config, self.outdir, self.srcdir) + filter_string = self.app.config.needs_builder_filter + needs_per_page_build_path = self.app.config.needs_per_page_build_path + filtered_needs = filter_needs(self.app, needs, filter_string) + needs_per_page_dir = os.path.join(self.outdir, needs_per_page_build_path) + + needs_per_page_data = {} + if not os.path.exists(needs_per_page_dir): + os.makedirs(needs_per_page_dir, exist_ok=True) + + # Create list docname-based dict has key is docname and value is list of need with same the docname . + needs_per_page_data_key = [] + for need in filtered_needs: + needs_id_dict = {} + id = need["id"] + needs_id_dict[id] = needs_list.make_simple_need(need) + docs_name = need.get("docname") + if docs_name in needs_per_page_data: + # add key docs_name + needs_per_page_data[docs_name].append(needs_id_dict) + else: + needs_per_page_data[docs_name] = [needs_id_dict] + + # create seperate file for every docname-based dict + needs_per_page_data_key = needs_per_page_data.keys() + for docs_name_key in needs_per_page_data_key: + docs_name = f"{docs_name_key}.json" + docs_name_file = os.path.join(needs_per_page_dir, docs_name) + docs_name_file_dir = os.path.dirname(docs_name_file) + if not os.path.exists(docs_name_file_dir): + os.mkdir(docs_name_file_dir) + try: + with open(docs_name_file, "w") as f: + data = {"needs": needs_per_page_data[docs_name_key]} + json.dump(data, f, indent=4) + + except Exception as e: + log.error(f"Needs-per-page: {docs_name_key} - error: {e}") + + log.info("needs per page successfully exported") + + def get_outdated_docs(self) -> Iterable[str]: + return [] + + def prepare_writing(self, _docnames: Set[str]) -> None: + pass + + def write_doc_serialized(self, _docname: str, _doctree: nodes.document) -> None: + pass + + def cleanup(self) -> None: + pass + + def get_target_uri(self, _docname: str, _typ: Optional[str] = None) -> str: + return "" + + +def build_needs_per_page_json(app: Sphinx, _exception: Exception) -> None: + env = unwrap(app.env) + if not env.config.needs_per_page: + return + + # Do not create an additional needs_json for every needs_id, if builder is already "needs_id". + if isinstance(app.builder, NeedsPerPageBuilder): + return + + try: + needs_per_page_builder = NeedsPerPageBuilder(app, env) + except TypeError: + needs_per_page_builder = NeedsPerPageBuilder(app) + needs_per_page_builder.set_environment(env) + + needs_per_page_builder.finish() diff --git a/sphinx_needs/needs.py b/sphinx_needs/needs.py index aa18f4750..2caec11b3 100644 --- a/sphinx_needs/needs.py +++ b/sphinx_needs/needs.py @@ -12,8 +12,10 @@ from sphinx_needs.api.configuration import add_extra_option from sphinx_needs.builder import ( NeedsBuilder, + NeedsPerPageBuilder, NeedumlsBuilder, build_needs_json, + build_needs_per_page_json, build_needumls_pumls, ) from sphinx_needs.config import NEEDS_CONFIG @@ -141,6 +143,7 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.add_builder(NeedsBuilder) app.add_builder(NeedumlsBuilder) + app.add_builder(NeedsPerPageBuilder) app.add_config_value( "needs_types", [ @@ -281,6 +284,10 @@ def setup(app: Sphinx) -> Dict[str, Any]: # app.add_config_value("needs_debug_measurement", False, "html", types=[dict]) + # add config for needs id with the sames docs_name + app.add_config_value("needs_per_page", False, "html", types=[bool]) + app.add_config_value("needs_per_page_build_path", "need_per_page", "html") + # Define nodes app.add_node(Need, html=(html_visit, html_depart), latex=(latex_visit, latex_depart)) app.add_node( @@ -376,6 +383,8 @@ def setup(app: Sphinx) -> Dict[str, Any]: app.connect("env-updated", install_lib_static_files) app.connect("env-updated", install_permalink_file) + # + app.connect("build-finished", build_needs_per_page_json) # This should be called last, so that need-styles can override styles from used libraries app.connect("env-updated", install_styles_static_files) diff --git a/sphinx_needs/needsfile.py b/sphinx_needs/needsfile.py index b209d47f6..fe36748a9 100644 --- a/sphinx_needs/needsfile.py +++ b/sphinx_needs/needsfile.py @@ -131,6 +131,10 @@ def load_json(self, file) -> None: self.log.debug(f"needs.json file loaded: {file}") + def make_simple_need(self, need_info): + writable_need = {key: need_info[key] for key in need_info if key not in self.JSON_KEY_EXCLUSIONS_FILTERS} + return writable_need + class Errors: def __init__(self, schema_errors: List[Any]): diff --git a/tests/test_needs_per_page_builder.py b/tests/test_needs_per_page_builder.py new file mode 100644 index 000000000..c33324686 --- /dev/null +++ b/tests/test_needs_per_page_builder.py @@ -0,0 +1,39 @@ +import json + +import pytest + +from sphinx_needs.filter_common import filter_needs + + +@pytest.mark.parametrize( + "test_app", [{"buildername": "needs_per_page", "srcdir": "doc_test/doc_needs_builder"}], indirect=True +) +def test_doc_needs_per_page_builder(test_app): + import os + + from sphinx_needs.utils import unwrap + + app = test_app + app.build() + out_dir = app.outdir + env = unwrap(app.env) + needs = env.needs_all_needs.values() + filter_string = app.config.needs_builder_filter + filtered_needs = filter_needs(app, needs, filter_string) + needs_per_page_build_path = app.config.needs_per_page_build_path + needs_per_page_parent_path = os.path.join(out_dir, needs_per_page_build_path) + for need in filtered_needs: + need_id = need["id"] + need_docname = need["docname"] + need_docname_file = f"{need_docname}.json" + need_docname_path = os.path.join(needs_per_page_parent_path, need_docname_file) + # need_docname = os.path.dirname(docs_name_file) + assert os.path.exists(need_docname_path) + + with open(need_docname_path) as needs_file: + needs_file_content = needs_file.read() + needs_list = json.loads(needs_file_content) + assert isinstance(needs_list["needs"], list) + needs_of_file = needs_list["needs"] + need_id_list = set().union(*(d for d in needs_of_file)) + assert need_id in need_id_list