From cbcefa91fa12ec83233ee6498648c28b9acc4a76 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 2 May 2025 09:34:20 -0700 Subject: [PATCH 01/33] tc --- .jenkins/validate_tutorials_built.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.jenkins/validate_tutorials_built.py b/.jenkins/validate_tutorials_built.py index f5cd187dbc..5c9e60e90b 100644 --- a/.jenkins/validate_tutorials_built.py +++ b/.jenkins/validate_tutorials_built.py @@ -53,7 +53,6 @@ "intermediate_source/tensorboard_profiler_tutorial", # reenable after 2.0 release. "advanced_source/semi_structured_sparse", # reenable after 3303 is fixed. "intermediate_source/torchrec_intro_tutorial", # reenable after 3302 is fixe - "intermediate_source/memory_format_tutorial", # causes other tutorials like torch_logs fail. "state" issue, reseting dynamo didn't help ] def tutorial_source_dirs() -> List[Path]: From 55abc486027c42704f84a37d9d5973c81c1557d4 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 2 May 2025 09:59:08 -0700 Subject: [PATCH 02/33] tc --- intermediate_source/memory_format_tutorial.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/intermediate_source/memory_format_tutorial.py b/intermediate_source/memory_format_tutorial.py index 26bc5c9d53..b73f3c674e 100644 --- a/intermediate_source/memory_format_tutorial.py +++ b/intermediate_source/memory_format_tutorial.py @@ -341,18 +341,17 @@ def check_cl(*args, **kwargs): return check_cl -old_attrs = dict() +old_attrs = [] def attribute(m): - old_attrs[m] = dict() for i in dir(m): e = getattr(m, i) exclude_functions = ["is_cuda", "has_names", "numel", "stride", "Tensor", "is_contiguous", "__class__"] if i not in exclude_functions and not i.startswith("_") and "__call__" in dir(e): try: - old_attrs[m][i] = e setattr(m, i, check_wrapper(e)) + old_attrs.append((m, i, e)) except Exception as e: print(i) print(e) @@ -372,9 +371,8 @@ def attribute(m): ###################################################################### # Code below is to recover the attributes of torch. -for (m, attrs) in old_attrs.items(): - for (k, v) in attrs.items(): - setattr(m, k, v) +for m, i, e in reversed(old_attrs): + setattr(m, i, e) ###################################################################### # Work to do From 40aba9fd16fe4b60c464885580b493fc7208bbf9 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 11:31:25 -0700 Subject: [PATCH 03/33] tc --- conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf.py b/conf.py index 05cfa11ca1..6e0e7c2826 100644 --- a/conf.py +++ b/conf.py @@ -98,6 +98,8 @@ # -- Sphinx-gallery configuration -------------------------------------------- def reset_seeds(gallery_conf, fname): + import importlib + importlib.reload(torch) torch.cuda.empty_cache() torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False From bf543efe3be9670d18f33e09beaf8b3dc96c7398 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 12:38:18 -0700 Subject: [PATCH 04/33] tc --- conf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf.py b/conf.py index 6e0e7c2826..36c3d4439f 100644 --- a/conf.py +++ b/conf.py @@ -98,8 +98,6 @@ # -- Sphinx-gallery configuration -------------------------------------------- def reset_seeds(gallery_conf, fname): - import importlib - importlib.reload(torch) torch.cuda.empty_cache() torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False @@ -128,6 +126,7 @@ def reset_seeds(gallery_conf, fname): 'pypandoc': {'extra_args': ['--mathjax', '--toc'], 'filters': ['.jenkins/custom_pandoc_filter.py'], }, + 'parallel': True, } html_baseurl = 'https://pytorch.org/tutorials/' # needed for sphinx-sitemap From 896247d7cc82621d4df524b0e82224e48cb87cbf Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 13:06:54 -0700 Subject: [PATCH 05/33] tc --- .ci/docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/requirements.txt b/.ci/docker/requirements.txt index e6802cb045..c1975a9180 100644 --- a/.ci/docker/requirements.txt +++ b/.ci/docker/requirements.txt @@ -2,7 +2,7 @@ # Refer to ./jenkins/build.sh for tutorial build instructions sphinx==5.0.0 -sphinx-gallery==0.11.1 +sphinx-gallery==0.16.1 sphinx_design docutils==0.16 sphinx-copybutton From 83854a4dc52fedfb28f6cf324a054cdecca07dd1 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 14:30:01 -0700 Subject: [PATCH 06/33] tc --- .ci/docker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/requirements.txt b/.ci/docker/requirements.txt index c1975a9180..dc01ea0c76 100644 --- a/.ci/docker/requirements.txt +++ b/.ci/docker/requirements.txt @@ -2,7 +2,7 @@ # Refer to ./jenkins/build.sh for tutorial build instructions sphinx==5.0.0 -sphinx-gallery==0.16.1 +sphinx-gallery==0.19.0 sphinx_design docutils==0.16 sphinx-copybutton From 310991d2c89f4b75b2e787da5a465d281c2c4b74 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 15:20:26 -0700 Subject: [PATCH 07/33] tc --- custom_directives.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom_directives.py b/custom_directives.py index 388aa262e6..761fde4d84 100644 --- a/custom_directives.py +++ b/custom_directives.py @@ -88,8 +88,8 @@ def run(self): if 'intro' in self.options: intro = self.options['intro'][:195] + '...' else: - _, blocks = sphinx_gallery.gen_rst.split_code_and_text_blocks(abs_fname) - intro, _ = sphinx_gallery.gen_rst.extract_intro_and_title(abs_fname, blocks[0][1]) + _, blocks = sphinx_gallery.py_source_parser.split_code_and_text_blocks(abs_fname) + intro, _ = sphinx_gallery.get_rst.extract_intro_and_title(abs_fname, blocks[0][1]) thumbnail_rst = '' #sphinx_gallery.backreferences._thumbnail_div( From de264f714b3351e1ed86fe6bbd81c7471ddaf11e Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 6 May 2025 15:37:56 -0700 Subject: [PATCH 08/33] tc --- custom_directives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_directives.py b/custom_directives.py index 761fde4d84..3b22ec2d8c 100644 --- a/custom_directives.py +++ b/custom_directives.py @@ -89,7 +89,7 @@ def run(self): intro = self.options['intro'][:195] + '...' else: _, blocks = sphinx_gallery.py_source_parser.split_code_and_text_blocks(abs_fname) - intro, _ = sphinx_gallery.get_rst.extract_intro_and_title(abs_fname, blocks[0][1]) + intro, _ = sphinx_gallery.gen_rst.extract_intro_and_title(abs_fname, blocks[0][1]) thumbnail_rst = '' #sphinx_gallery.backreferences._thumbnail_div( From 56a30dad28a4c59825d9306c5a71ccda64a5adcb Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 09:33:28 -0700 Subject: [PATCH 09/33] tc --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index 36c3d4439f..ebd49b92bb 100644 --- a/conf.py +++ b/conf.py @@ -126,7 +126,7 @@ def reset_seeds(gallery_conf, fname): 'pypandoc': {'extra_args': ['--mathjax', '--toc'], 'filters': ['.jenkins/custom_pandoc_filter.py'], }, - 'parallel': True, + # 'parallel': True, } html_baseurl = 'https://pytorch.org/tutorials/' # needed for sphinx-sitemap From e60b1827414a185a318b4c56b1c721408e41a716 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 11:30:48 -0700 Subject: [PATCH 10/33] tc --- intermediate_source/memory_format_tutorial.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/intermediate_source/memory_format_tutorial.py b/intermediate_source/memory_format_tutorial.py index b73f3c674e..26bc5c9d53 100644 --- a/intermediate_source/memory_format_tutorial.py +++ b/intermediate_source/memory_format_tutorial.py @@ -341,17 +341,18 @@ def check_cl(*args, **kwargs): return check_cl -old_attrs = [] +old_attrs = dict() def attribute(m): + old_attrs[m] = dict() for i in dir(m): e = getattr(m, i) exclude_functions = ["is_cuda", "has_names", "numel", "stride", "Tensor", "is_contiguous", "__class__"] if i not in exclude_functions and not i.startswith("_") and "__call__" in dir(e): try: + old_attrs[m][i] = e setattr(m, i, check_wrapper(e)) - old_attrs.append((m, i, e)) except Exception as e: print(i) print(e) @@ -371,8 +372,9 @@ def attribute(m): ###################################################################### # Code below is to recover the attributes of torch. -for m, i, e in reversed(old_attrs): - setattr(m, i, e) +for (m, attrs) in old_attrs.items(): + for (k, v) in attrs.items(): + setattr(m, k, v) ###################################################################### # Work to do From 101d26e27cc6176792ba816c42cfbfd28ab174a5 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 13:05:59 -0700 Subject: [PATCH 11/33] tc --- conf.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/conf.py b/conf.py index ebd49b92bb..108d7c619e 100644 --- a/conf.py +++ b/conf.py @@ -98,18 +98,7 @@ # -- Sphinx-gallery configuration -------------------------------------------- def reset_seeds(gallery_conf, fname): - torch.cuda.empty_cache() - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - torch._dynamo.reset() - torch._inductor.config.force_disable_caches = True - torch.manual_seed(42) - torch.set_default_device(None) - random.seed(10) - numpy.random.seed(10) - torch.set_grad_enabled(True) - - gc.collect() + pass sphinx_gallery_conf = { 'examples_dirs': ['beginner_source', 'intermediate_source', From 94d489e8647cf813a73fc56be40c550e7c913787 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 13:42:05 -0700 Subject: [PATCH 12/33] tc --- conf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/conf.py b/conf.py index 108d7c619e..ec62c8cee7 100644 --- a/conf.py +++ b/conf.py @@ -48,6 +48,17 @@ from pathlib import Path pio.renderers.default = 'sphinx_gallery' +import multiprocessing as mp + +import sphinx_gallery.gen_rst +original_generate_file_rst = sphinx_gallery.gen_rst.generate_file_rst +def generate_file_rst(fname, target_dir, src_dir, gallery_conf): + pool = mp.Pool(1) + p = pool.apply_async(original_generate_file_rst, (fname, target_dir, src_dir, gallery_conf)) + pool.close() + pool.join() + return p.get() +sphinx_gallery.gen_rst.generate_file_rst = generate_file_rst try: import torchvision From fe157ffd3995b8819425b3c71d78f85d0a9cfd56 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 13:59:22 -0700 Subject: [PATCH 13/33] tc --- conf.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/conf.py b/conf.py index ec62c8cee7..c252f75c7d 100644 --- a/conf.py +++ b/conf.py @@ -48,18 +48,6 @@ from pathlib import Path pio.renderers.default = 'sphinx_gallery' -import multiprocessing as mp - -import sphinx_gallery.gen_rst -original_generate_file_rst = sphinx_gallery.gen_rst.generate_file_rst -def generate_file_rst(fname, target_dir, src_dir, gallery_conf): - pool = mp.Pool(1) - p = pool.apply_async(original_generate_file_rst, (fname, target_dir, src_dir, gallery_conf)) - pool.close() - pool.join() - return p.get() -sphinx_gallery.gen_rst.generate_file_rst = generate_file_rst - try: import torchvision except ImportError: @@ -126,7 +114,7 @@ def reset_seeds(gallery_conf, fname): 'pypandoc': {'extra_args': ['--mathjax', '--toc'], 'filters': ['.jenkins/custom_pandoc_filter.py'], }, - # 'parallel': True, + 'parallel': True, } html_baseurl = 'https://pytorch.org/tutorials/' # needed for sphinx-sitemap From 43704cb6da16f20526db3fcfc5edd76aaeaa6656 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 14:21:17 -0700 Subject: [PATCH 14/33] tc --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index c252f75c7d..fa1863b68d 100644 --- a/conf.py +++ b/conf.py @@ -114,7 +114,7 @@ def reset_seeds(gallery_conf, fname): 'pypandoc': {'extra_args': ['--mathjax', '--toc'], 'filters': ['.jenkins/custom_pandoc_filter.py'], }, - 'parallel': True, + 'parallel': 3, } html_baseurl = 'https://pytorch.org/tutorials/' # needed for sphinx-sitemap From 9d45f123cb986a050e18517a5cb032c03548448b Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 7 May 2025 20:40:51 -0700 Subject: [PATCH 15/33] tc --- conf.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/conf.py b/conf.py index fa1863b68d..cd0c582e71 100644 --- a/conf.py +++ b/conf.py @@ -48,6 +48,36 @@ from pathlib import Path pio.renderers.default = 'sphinx_gallery' +import sphinx_gallery.gen_rst +import multiprocessing + +# Save the original function +def isolated_call(func, args, kwargs, result_queue): + try: + result = func(*args, **kwargs) + result_queue.put((True, result)) + except Exception as e: + result_queue.put((False, str(e))) + +def make_isolated_version(func): + def wrapper(*args, **kwargs): + result_queue = multiprocessing.Queue() + p = multiprocessing.Process( + target=isolated_call, + args=(func, args, kwargs, result_queue) + ) + p.start() + p.join() + success, result = result_queue.get() + if success: + return result + else: + raise RuntimeError(f"Error in isolated process: {result}") + return wrapper + +# Monkey-patch +sphinx_gallery.gen_rst.generate_file_rst = make_isolated_version(sphinx_gallery.gen_rst.generate_file_rst) + try: import torchvision except ImportError: @@ -114,7 +144,6 @@ def reset_seeds(gallery_conf, fname): 'pypandoc': {'extra_args': ['--mathjax', '--toc'], 'filters': ['.jenkins/custom_pandoc_filter.py'], }, - 'parallel': 3, } html_baseurl = 'https://pytorch.org/tutorials/' # needed for sphinx-sitemap From ff57d614192fba721c5dcff62ffe5b6897854a58 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 9 May 2025 15:02:33 -0700 Subject: [PATCH 16/33] tc --- .ci/docker/requirements.txt | 2 +- .jenkins/build.sh | 5 ++++- .jenkins/sphinx_files.py | 13 +++++++++++++ Makefile | 12 ++++++++---- conf.py | 31 +------------------------------ custom_directives.py | 2 +- 6 files changed, 28 insertions(+), 37 deletions(-) create mode 100644 .jenkins/sphinx_files.py diff --git a/.ci/docker/requirements.txt b/.ci/docker/requirements.txt index dc01ea0c76..e6802cb045 100644 --- a/.ci/docker/requirements.txt +++ b/.ci/docker/requirements.txt @@ -2,7 +2,7 @@ # Refer to ./jenkins/build.sh for tutorial build instructions sphinx==5.0.0 -sphinx-gallery==0.19.0 +sphinx-gallery==0.11.1 sphinx_design docutils==0.16 sphinx-copybutton diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 58483c168b..976f757a15 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -58,7 +58,10 @@ if [[ "${JOB_TYPE}" == "worker" ]]; then # Step 3: Run `make docs` to generate HTML files and static files for these tutorialis pip3 install -e git+https://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme - make docs + make download + make download-last-reviewed-json + python .jenkins/sphinx_files.py + make postprocess # Step 3.1: Run the post-processing script: python .jenkins/post_process_notebooks.py diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py new file mode 100644 index 0000000000..b280c369fd --- /dev/null +++ b/.jenkins/sphinx_files.py @@ -0,0 +1,13 @@ +import subprocess +import os + +def main() -> None: + files_to_run = os.environ["FILES_TO_RUN"] + env = os.environ.copy() + for file in files_to_run.split(" "): + print(f"Running {file}") + env["RUNTHIS"] = file + subprocess.check_output(["make", "html"], env=env) + +if __name__ == "__main__": + main() diff --git a/Makefile b/Makefile index 9068d32b2a..1e1f6fdb82 100644 --- a/Makefile +++ b/Makefile @@ -90,16 +90,20 @@ download-last-reviewed-json: @echo "Downloading tutorials-review-data.json..." curl -o tutorials-review-data.json https://raw.githubusercontent.com/pytorch/tutorials/refs/heads/last-reviewed-data-json/tutorials-review-data.json @echo "Finished downloading tutorials-review-data.json." -docs: - make download - make download-last-reviewed-json - make html + +postprocess: @python .jenkins/insert_last_verified.py $(BUILDDIR)/html rm -rf docs cp -r $(BUILDDIR)/html docs touch docs/.nojekyll rm -rf tutorials-review-data.json +docs: + make download + make download-last-reviewed-json + make html + make postprocess + html-noplot: $(SPHINXBUILD) -D plot_gallery=0 -b html $(SPHINXOPTS) "$(SOURCEDIR)" "$(BUILDDIR)/html" # bash .jenkins/remove_invisible_code_block_batch.sh "$(BUILDDIR)/html" diff --git a/conf.py b/conf.py index cd0c582e71..f89ee56d77 100644 --- a/conf.py +++ b/conf.py @@ -48,35 +48,6 @@ from pathlib import Path pio.renderers.default = 'sphinx_gallery' -import sphinx_gallery.gen_rst -import multiprocessing - -# Save the original function -def isolated_call(func, args, kwargs, result_queue): - try: - result = func(*args, **kwargs) - result_queue.put((True, result)) - except Exception as e: - result_queue.put((False, str(e))) - -def make_isolated_version(func): - def wrapper(*args, **kwargs): - result_queue = multiprocessing.Queue() - p = multiprocessing.Process( - target=isolated_call, - args=(func, args, kwargs, result_queue) - ) - p.start() - p.join() - success, result = result_queue.get() - if success: - return result - else: - raise RuntimeError(f"Error in isolated process: {result}") - return wrapper - -# Monkey-patch -sphinx_gallery.gen_rst.generate_file_rst = make_isolated_version(sphinx_gallery.gen_rst.generate_file_rst) try: import torchvision @@ -133,7 +104,7 @@ def reset_seeds(gallery_conf, fname): 'examples_dirs': ['beginner_source', 'intermediate_source', 'advanced_source', 'recipes_source', 'prototype_source'], 'gallery_dirs': ['beginner', 'intermediate', 'advanced', 'recipes', 'prototype'], - 'filename_pattern': re.compile(SPHINX_SHOULD_RUN), + 'filename_pattern': os.getenv("RUNTHIS"), 'promote_jupyter_magic': True, 'backreferences_dir': None, 'first_notebook_cell': ("# For tips on running notebooks in Google Colab, see\n" diff --git a/custom_directives.py b/custom_directives.py index 3b22ec2d8c..388aa262e6 100644 --- a/custom_directives.py +++ b/custom_directives.py @@ -88,7 +88,7 @@ def run(self): if 'intro' in self.options: intro = self.options['intro'][:195] + '...' else: - _, blocks = sphinx_gallery.py_source_parser.split_code_and_text_blocks(abs_fname) + _, blocks = sphinx_gallery.gen_rst.split_code_and_text_blocks(abs_fname) intro, _ = sphinx_gallery.gen_rst.extract_intro_and_title(abs_fname, blocks[0][1]) thumbnail_rst = '' From 4fa9d57dd70a03b97467894696ab51fec75ca689 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 9 May 2025 15:38:33 -0700 Subject: [PATCH 17/33] tc --- .jenkins/get_files_to_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py index bdf4562a82..340900c004 100644 --- a/.jenkins/get_files_to_run.py +++ b/.jenkins/get_files_to_run.py @@ -96,6 +96,7 @@ def main() -> None: all_files = get_all_files() files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num - 1] + files_to_run = [x for x in files_to_run if x not in NOT_RUN] if not args.dry_run: remove_other_files(all_files, compute_files_to_keep(files_to_run)) stripped_file_names = [Path(x).stem for x in files_to_run] From 87b9f689f1de7eca856576b684ef03b94753645c Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Fri, 9 May 2025 15:47:12 -0700 Subject: [PATCH 18/33] tc --- .jenkins/get_files_to_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py index 340900c004..9a75b933a4 100644 --- a/.jenkins/get_files_to_run.py +++ b/.jenkins/get_files_to_run.py @@ -3,6 +3,7 @@ import os from pathlib import Path from remove_runnable_code import remove_runnable_code +from validate_tutorials_built import NOT_RUN # Calculate repo base dir From a0fb48099c0c4051aca653e543748a1020b3cead Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Sat, 10 May 2025 09:50:06 -0700 Subject: [PATCH 19/33] tc --- .jenkins/get_files_to_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py index 9a75b933a4..3ec72dc384 100644 --- a/.jenkins/get_files_to_run.py +++ b/.jenkins/get_files_to_run.py @@ -97,7 +97,7 @@ def main() -> None: all_files = get_all_files() files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num - 1] - files_to_run = [x for x in files_to_run if x not in NOT_RUN] + files_to_run = [x for x in files_to_run if f"{x}.py" not in NOT_RUN] if not args.dry_run: remove_other_files(all_files, compute_files_to_keep(files_to_run)) stripped_file_names = [Path(x).stem for x in files_to_run] From f433a269b3bf4cdf895f561c94f0330650a6eb9c Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Sat, 10 May 2025 10:57:17 -0700 Subject: [PATCH 20/33] tc --- .jenkins/get_files_to_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/get_files_to_run.py b/.jenkins/get_files_to_run.py index 3ec72dc384..38a5d964c8 100644 --- a/.jenkins/get_files_to_run.py +++ b/.jenkins/get_files_to_run.py @@ -97,7 +97,7 @@ def main() -> None: all_files = get_all_files() files_to_run = calculate_shards(all_files, num_shards=args.num_shards)[args.shard_num - 1] - files_to_run = [x for x in files_to_run if f"{x}.py" not in NOT_RUN] + files_to_run = [x for x in files_to_run if x not in [f"{f}.py" for f in NOT_RUN]] if not args.dry_run: remove_other_files(all_files, compute_files_to_keep(files_to_run)) stripped_file_names = [Path(x).stem for x in files_to_run] From 7fc9b45cd58b1bc9fcdf9de0b143500e04951348 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 09:06:21 -0700 Subject: [PATCH 21/33] tc --- .jenkins/sphinx_files.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index b280c369fd..9d9ff2617d 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -1,13 +1,31 @@ +import glob +from pathlib import Path +import shutil import subprocess import os +from get_files_to_run import remove_other_files, compute_files_to_keep, calculate_shards, get_all_files +from validate_tutorials_built import NOT_RUN def main() -> None: - files_to_run = os.environ["FILES_TO_RUN"] + all_files = get_all_files() + files_to_run = calculate_shards(all_files, num_shards=20)[int(os.environ.get("WORKER_ID", "1")) - 1] + files_to_run = [x for x in files_to_run if x not in [f"{f}.py" for f in NOT_RUN]] + + os.mkdir("docs_to_zip", exist_ok=True) + env = os.environ.copy() - for file in files_to_run.split(" "): + for file in files_to_run: + remove_other_files(all_files, compute_files_to_keep(file)) print(f"Running {file}") - env["RUNTHIS"] = file + stem = Path(file).stem + env["RUNTHIS"] = stem subprocess.check_output(["make", "html"], env=env) + for file in glob.glob(f"docs/**/*", recursive=True): + if stem in file: + shutil.copy(file, f"docs_to_zip/{file}") + subprocess.check_output(["git", "reset", "--hard", "HEAD"]) + os.remove("docs") + shutil.move("docs_to_zip", "docs") if __name__ == "__main__": main() From 9b5cde46af304d5a1c5a13c01c7f1c5b32732f96 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 10:43:57 -0700 Subject: [PATCH 22/33] tc --- .jenkins/sphinx_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 9d9ff2617d..e6c8542251 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -11,7 +11,7 @@ def main() -> None: files_to_run = calculate_shards(all_files, num_shards=20)[int(os.environ.get("WORKER_ID", "1")) - 1] files_to_run = [x for x in files_to_run if x not in [f"{f}.py" for f in NOT_RUN]] - os.mkdir("docs_to_zip", exist_ok=True) + os.makedirs("docs_to_zip", exist_ok=True) env = os.environ.copy() for file in files_to_run: From 038e8172646bc734805478472191f78589838071 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 11:47:19 -0700 Subject: [PATCH 23/33] tc --- .jenkins/sphinx_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index e6c8542251..f137762fd6 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -20,6 +20,7 @@ def main() -> None: stem = Path(file).stem env["RUNTHIS"] = stem subprocess.check_output(["make", "html"], env=env) + subprocess.check_output(["make", "postprocess"], env=env) for file in glob.glob(f"docs/**/*", recursive=True): if stem in file: shutil.copy(file, f"docs_to_zip/{file}") From 87ea351fa5f59a68c689b9e75835eef22b4e10c2 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 13:45:44 -0700 Subject: [PATCH 24/33] tc --- .jenkins/sphinx_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index f137762fd6..f30832caf6 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -23,6 +23,7 @@ def main() -> None: subprocess.check_output(["make", "postprocess"], env=env) for file in glob.glob(f"docs/**/*", recursive=True): if stem in file: + os.makedirs(os.path.dirname(f"docs_to_zip/{file}"), exist_ok=True) shutil.copy(file, f"docs_to_zip/{file}") subprocess.check_output(["git", "reset", "--hard", "HEAD"]) os.remove("docs") From 895c3ce9b23b02150a1f150bd42b1d3a3f4e3bb4 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 14:16:58 -0700 Subject: [PATCH 25/33] tc --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1d9d572e56..29c60912f6 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ cleanup.sh # pyspelling dictionary.dic + +# CI stuff +tutorials-review-data.json From aa5b662101b98ffb3d26fa9579290ec118f1692c Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 14:59:34 -0700 Subject: [PATCH 26/33] tc --- .jenkins/sphinx_files.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index f30832caf6..fa845898ad 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -11,7 +11,7 @@ def main() -> None: files_to_run = calculate_shards(all_files, num_shards=20)[int(os.environ.get("WORKER_ID", "1")) - 1] files_to_run = [x for x in files_to_run if x not in [f"{f}.py" for f in NOT_RUN]] - os.makedirs("docs_to_zip", exist_ok=True) + os.makedirs("/tmp/docs_to_zip", exist_ok=True) env = os.environ.copy() for file in files_to_run: @@ -23,11 +23,11 @@ def main() -> None: subprocess.check_output(["make", "postprocess"], env=env) for file in glob.glob(f"docs/**/*", recursive=True): if stem in file: - os.makedirs(os.path.dirname(f"docs_to_zip/{file}"), exist_ok=True) - shutil.copy(file, f"docs_to_zip/{file}") + relative_path = Path(os.path.relpath(file, "docs")) + os.makedirs(os.path.dirname(f"docs_to_zip/{relative_path.parent}"), exist_ok=True) + shutil.copy(file, f"/tmp/docs_to_zip/{relative_path}") subprocess.check_output(["git", "reset", "--hard", "HEAD"]) - os.remove("docs") - shutil.move("docs_to_zip", "docs") + shutil.move("/tmp/docs_to_zip", "docs") if __name__ == "__main__": main() From cc0dde6cf4e4988f549b377966e9a4fa0959d8b3 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 15:39:27 -0700 Subject: [PATCH 27/33] tc --- .jenkins/sphinx_files.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index fa845898ad..15900ff93e 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -21,10 +21,13 @@ def main() -> None: env["RUNTHIS"] = stem subprocess.check_output(["make", "html"], env=env) subprocess.check_output(["make", "postprocess"], env=env) + print("Done running") for file in glob.glob(f"docs/**/*", recursive=True): if stem in file: relative_path = Path(os.path.relpath(file, "docs")) - os.makedirs(os.path.dirname(f"docs_to_zip/{relative_path.parent}"), exist_ok=True) + print(relative_path) + print(relative_path.parent) + os.makedirs(os.path.dirname(f"/tmp/docs_to_zip/{relative_path.parent}"), exist_ok=True) shutil.copy(file, f"/tmp/docs_to_zip/{relative_path}") subprocess.check_output(["git", "reset", "--hard", "HEAD"]) shutil.move("/tmp/docs_to_zip", "docs") From eda0644161f568754ac669b99339a71871d09125 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 16:01:21 -0700 Subject: [PATCH 28/33] tc --- .github/workflows/build-tutorials.yml | 14 -------------- .jenkins/sphinx_files.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/build-tutorials.yml b/.github/workflows/build-tutorials.yml index 94cfd5843a..7968cb118a 100644 --- a/.github/workflows/build-tutorials.yml +++ b/.github/workflows/build-tutorials.yml @@ -16,21 +16,7 @@ jobs: strategy: matrix: include: - - { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" } - { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } fail-fast: false runs-on: ${{ matrix.runner }} steps: diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 15900ff93e..4c645a0eb6 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -27,7 +27,7 @@ def main() -> None: relative_path = Path(os.path.relpath(file, "docs")) print(relative_path) print(relative_path.parent) - os.makedirs(os.path.dirname(f"/tmp/docs_to_zip/{relative_path.parent}"), exist_ok=True) + os.makedirs(os.path.dirname(f"/tmp/docs_to_zip/{relative_path}"), exist_ok=True) shutil.copy(file, f"/tmp/docs_to_zip/{relative_path}") subprocess.check_output(["git", "reset", "--hard", "HEAD"]) shutil.move("/tmp/docs_to_zip", "docs") From 87d686708e921ad4e4e57bb57ddb4a88bd0167b9 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 16:30:52 -0700 Subject: [PATCH 29/33] tc --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 1e1f6fdb82..7721e81a19 100644 --- a/Makefile +++ b/Makefile @@ -96,7 +96,6 @@ postprocess: rm -rf docs cp -r $(BUILDDIR)/html docs touch docs/.nojekyll - rm -rf tutorials-review-data.json docs: make download From f2c48010c7db0dd73a90acbab547e4afeb77f964 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 20:40:52 -0700 Subject: [PATCH 30/33] tc --- .github/workflows/build-tutorials.yml | 14 ++++++++++++++ .jenkins/sphinx_files.py | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-tutorials.yml b/.github/workflows/build-tutorials.yml index 7968cb118a..94cfd5843a 100644 --- a/.github/workflows/build-tutorials.yml +++ b/.github/workflows/build-tutorials.yml @@ -16,7 +16,21 @@ jobs: strategy: matrix: include: + - { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" } - { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } + - { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } fail-fast: false runs-on: ${{ matrix.runner }} steps: diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 4c645a0eb6..18803885a0 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -19,7 +19,8 @@ def main() -> None: print(f"Running {file}") stem = Path(file).stem env["RUNTHIS"] = stem - subprocess.check_output(["make", "html"], env=env) + result = subprocess.check_output(["make", "html"], env=env) + print(result.decode("utf-8")) subprocess.check_output(["make", "postprocess"], env=env) print("Done running") for file in glob.glob(f"docs/**/*", recursive=True): From 361bcd47d666acef61e8dad1c70042e19d44f401 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 21:39:51 -0700 Subject: [PATCH 31/33] tc --- .github/workflows/build-tutorials.yml | 14 -------------- .jenkins/sphinx_files.py | 2 +- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/build-tutorials.yml b/.github/workflows/build-tutorials.yml index 94cfd5843a..c45ec49f69 100644 --- a/.github/workflows/build-tutorials.yml +++ b/.github/workflows/build-tutorials.yml @@ -16,21 +16,7 @@ jobs: strategy: matrix: include: - - { shard: 1, num_shards: 15, runner: "linux.g5.12xlarge.nvidia.gpu" } - - { shard: 2, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 3, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 4, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 5, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - { shard: 6, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 7, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 8, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 9, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 10, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 11, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 12, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 13, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 14, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } - - { shard: 15, num_shards: 15, runner: "linux.g5.4xlarge.nvidia.gpu" } fail-fast: false runs-on: ${{ matrix.runner }} steps: diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 18803885a0..78c41bebee 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -15,7 +15,7 @@ def main() -> None: env = os.environ.copy() for file in files_to_run: - remove_other_files(all_files, compute_files_to_keep(file)) + remove_other_files(all_files, compute_files_to_keep([file])) print(f"Running {file}") stem = Path(file).stem env["RUNTHIS"] = stem From 66a99982ac7996b66e5434089a2def33258cc7d5 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 22:29:08 -0700 Subject: [PATCH 32/33] tc --- .jenkins/sphinx_files.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index 78c41bebee..d8469b261d 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -31,6 +31,8 @@ def main() -> None: os.makedirs(os.path.dirname(f"/tmp/docs_to_zip/{relative_path}"), exist_ok=True) shutil.copy(file, f"/tmp/docs_to_zip/{relative_path}") subprocess.check_output(["git", "reset", "--hard", "HEAD"]) + for file in glob.glob("/tmp/docs_to_zip", recursive=True): + print(file) shutil.move("/tmp/docs_to_zip", "docs") if __name__ == "__main__": From d59be8753c39af8d2706a494a455de23a359b3eb Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 13 May 2025 22:30:18 -0700 Subject: [PATCH 33/33] tc --- .jenkins/sphinx_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/sphinx_files.py b/.jenkins/sphinx_files.py index d8469b261d..cbaa6440cd 100644 --- a/.jenkins/sphinx_files.py +++ b/.jenkins/sphinx_files.py @@ -33,7 +33,7 @@ def main() -> None: subprocess.check_output(["git", "reset", "--hard", "HEAD"]) for file in glob.glob("/tmp/docs_to_zip", recursive=True): print(file) - shutil.move("/tmp/docs_to_zip", "docs") + shutil.move("/tmp/docs_to_zip", "_build/html") if __name__ == "__main__": main()