diff --git a/var/spack/repos/builtin/packages/py-torch/package.py b/var/spack/repos/builtin/packages/py-torch/package.py
index 7329ba91741f6553f373ae350d5f62b51a478d34..b2b15c7dfc2e3a018b70123a355ee64800b32e7a 100644
--- a/var/spack/repos/builtin/packages/py-torch/package.py
+++ b/var/spack/repos/builtin/packages/py-torch/package.py
@@ -107,9 +107,6 @@ class PyTorch(PythonPackage, CudaPackage):
     conflicts('+tbb', when='@:1.1')
     # https://github.com/pytorch/pytorch/issues/35149
     conflicts('+fbgemm', when='@1.4.0')
-    # https://github.com/pytorch/pytorch/issues/35478
-    conflicts('%clang@11.0.3-apple',
-              msg='Apple Clang 11.0.3 segfaults at build-time')
 
     conflicts('cuda_arch=none', when='+cuda',
               msg='Must specify CUDA compute capabilities of your GPU, see '
@@ -170,6 +167,17 @@ class PyTorch(PythonPackage, CudaPackage):
     depends_on('py-six', type='test')
     depends_on('py-psutil', type='test')
 
+    # https://github.com/pytorch/pytorch/pull/35607
+    # https://github.com/pytorch/pytorch/pull/37865
+    # Fixes CMake configuration error when XNNPACK is disabled
+    patch('xnnpack.patch', when='@1.5.0')
+
+    # https://github.com/pytorch/pytorch/pull/37086
+    # Fixes compilation with Clang 9.0.0 and Apple Clang 11.0.3
+    patch('https://github.com/pytorch/pytorch/commit/e921cd222a8fbeabf5a3e74e83e0d8dfb01aa8b5.patch',
+          sha256='7781c7ec0a661bf5a946a659f80e90df9dba116ad168762f15b10547113ae600',
+          when='@1.1:1.5')
+
     # Both build and install run cmake/make/make install
     # Only run once to speed up build times
     phases = ['install']
diff --git a/var/spack/repos/builtin/packages/py-torch/xnnpack.patch b/var/spack/repos/builtin/packages/py-torch/xnnpack.patch
new file mode 100644
index 0000000000000000000000000000000000000000..154033081e7ff91867e9a043a93c46b888bfe8cb
--- /dev/null
+++ b/var/spack/repos/builtin/packages/py-torch/xnnpack.patch
@@ -0,0 +1,47 @@
+diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
+index 8025a7de3c..0da37079d6 100644
+--- a/caffe2/CMakeLists.txt
++++ b/caffe2/CMakeLists.txt
+@@ -46,12 +46,19 @@ if (INTERN_BUILD_ATEN_OPS)
+   list(APPEND Caffe2_DEPENDENCY_INCLUDE ${ATen_THIRD_PARTY_INCLUDE})
+ endif()
+ 
++# {Q/X,etc} NPACK support is enabled by default, if none of these options
++# are selected, turn this flag ON to incidate the support is disabled
++set(NNPACK_AND_FAMILY_DISABLED OFF)
++if(NOT (USE_NNPACK OR USE_QNNPACK OR USE_PYTORCH_QNNPACK OR USE_XNNPACK))
++  set(NNPACK_AND_FAMILY_DISABLED ON)
++endif()
++
+ # ---[ Caffe2 build
+ # Note: the folders that are being commented out have not been properly
+ # addressed yet.
+ 
+ # For pthreadpool_new_if_impl. TODO: Remove when threadpools are unitied.
+-if (NOT MSVC)
++if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
+   IF(NOT TARGET fxdiv)
+     SET(FXDIV_BUILD_TESTS OFF CACHE BOOL "")
+     SET(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "")
+@@ -710,7 +717,7 @@ ELSEIF(USE_CUDA)
+ ENDIF()
+ 
+ 
+-if (NOT MSVC)
++if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
+   TARGET_LINK_LIBRARIES(torch_cpu PRIVATE fxdiv)
+ endif()
+ 
+diff --git a/caffe2/utils/CMakeLists.txt b/caffe2/utils/CMakeLists.txt
+index 27aabb1315..3c7845c67d 100644
+--- a/caffe2/utils/CMakeLists.txt
++++ b/caffe2/utils/CMakeLists.txt
+@@ -36,7 +36,7 @@ list(APPEND Caffe2_CPU_SRCS
+ # ---[ threadpool/pthreadpool* is a local modification of the NNPACK
+ # pthreadpool with a very similar interface. Neither NNPACK, nor this
+ # thread pool supports Windows.
+-if (NOT MSVC)
++if (NOT MSVC AND NOT NNPACK_AND_FAMILY_DISABLED)
+   add_definitions(-DUSE_INTERNAL_THREADPOOL_IMPL)
+   set(Caffe2_CPU_SRCS ${Caffe2_CPU_SRCS}
+           utils/threadpool/pthreadpool.cc