Intrepid2_ProjectionTools.hpp

Go to the documentation of this file.

// @HEADER
// ************************************************************************
//
//                           Intrepid2 Package
//                 Copyright (2007) Sandia Corporation
//
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Kyungjoo Kim  (kyukim@sandia.gov), or
//                    Mauro Perego  (mperego@sandia.gov)
//
// ************************************************************************
// @HEADER

#ifndef __INTREPID2_PROJECTIONTOOLS_HPP__
#define __INTREPID2_PROJECTIONTOOLS_HPP__

#include "Intrepid2_ConfigDefs.hpp"
#include "Intrepid2_Types.hpp"
#include "Intrepid2_Utils.hpp"

#include "Shards_CellTopology.hpp"
#include "Shards_BasicTopologies.hpp"

#include "Intrepid2_PointTools.hpp"

#include "Intrepid2_Basis.hpp"

// -- HGRAD family
#include "Intrepid2_HGRAD_LINE_Cn_FEM.hpp"
#include "Intrepid2_HGRAD_QUAD_Cn_FEM.hpp"
#include "Intrepid2_HGRAD_HEX_Cn_FEM.hpp"

#include "Intrepid2_HGRAD_TRI_Cn_FEM.hpp"
#include "Intrepid2_HGRAD_TET_Cn_FEM.hpp"

// -- HCURL family
#include "Intrepid2_HCURL_QUAD_In_FEM.hpp"
#include "Intrepid2_HCURL_HEX_In_FEM.hpp"

#include "Intrepid2_HCURL_TRI_In_FEM.hpp"
#include "Intrepid2_HCURL_TET_In_FEM.hpp"
#include "Intrepid2_HVOL_LINE_Cn_FEM.hpp"

// -- HDIV family
#include "Intrepid2_HDIV_QUAD_In_FEM.hpp"
#include "Intrepid2_HDIV_HEX_In_FEM.hpp"

#include "Intrepid2_HDIV_TRI_In_FEM.hpp"
#include "Intrepid2_HDIV_TET_In_FEM.hpp"
#include "Intrepid2_HVOL_TRI_Cn_FEM.hpp"

// -- Lower order family
#include "Intrepid2_HCURL_QUAD_I1_FEM.hpp"
#include "Intrepid2_HCURL_TRI_I1_FEM.hpp"

#include "Intrepid2_HDIV_QUAD_I1_FEM.hpp"
#include "Intrepid2_HDIV_TRI_I1_FEM.hpp"

#include "Intrepid2_HCURL_HEX_I1_FEM.hpp"
#include "Intrepid2_HCURL_TET_I1_FEM.hpp"
#include "Intrepid2_HCURL_WEDGE_I1_FEM.hpp"

#include "Intrepid2_HDIV_HEX_I1_FEM.hpp"
#include "Intrepid2_HDIV_TET_I1_FEM.hpp"
#include "Intrepid2_HDIV_WEDGE_I1_FEM.hpp"

#include "Teuchos_LAPACK.hpp"
#include "Intrepid2_OrientationTools.hpp"

#include "Intrepid2_ProjectionStruct.hpp"

#ifdef HAVE_INTREPID2_KOKKOSKERNELS
#include "KokkosBatched_QR_Serial_Internal.hpp"
#include "KokkosBatched_ApplyQ_Serial_Internal.hpp"
#include "KokkosBatched_Trsv_Serial_Internal.hpp"
#include "KokkosBatched_Util.hpp"
#endif

namespace Intrepid2 {

namespace Experimental {



template<typename DeviceType>
class ProjectionTools {
public:
  using ExecSpaceType = typename DeviceType::execution_space;
  using MemSpaceType = typename DeviceType::memory_space;
  using EvalPointsType = typename ProjectionStruct<DeviceType, double>::EvalPointsType;


  template<typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getL2EvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<typename basisCoeffsValueType, class ...basisCoeffsProperties,
  typename funValsValueType, class ...funValsProperties,
  typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getL2BasisCoeffs(Kokkos::DynRankView<basisCoeffsValueType,basisCoeffsProperties...> basisCoeffs,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetAtEvalPoints,
      const typename BasisType::ScalarViewType evaluationPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);


  template<typename BasisType>
  static void
  getL2DGEvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<typename basisCoeffsValueType, class ...basisCoeffsProperties,
  typename funValsValueType, class ...funValsProperties,
  typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getL2DGBasisCoeffs(Kokkos::DynRankView<basisCoeffsValueType,basisCoeffsProperties...> basisCoeffs,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetAtEvalPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);

  template<typename basisViewType, typename targetViewType, typename BasisType>
  static void
  getL2DGBasisCoeffs(basisViewType basisCoeffs,
      const targetViewType targetAtTargetEPoints,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);


  template<typename BasisType, typename OrientationViewType >
  static void
  getHGradEvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      typename BasisType::ScalarViewType gradEvalPoints,
      const OrientationViewType cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<class BasisCoeffsViewType, class TargetValueViewType, class TargetGradViewType,
           class BasisType, class OrientationViewType>
  static void
  getHGradBasisCoeffs(BasisCoeffsViewType basisCoeffs,
                      const TargetValueViewType targetAtEvalPoints,
                      const TargetGradViewType targetGradAtGradEvalPoints,
                      const typename BasisType::ScalarViewType evaluationPoints,
                      const typename BasisType::ScalarViewType gradEvalPoints,
                      const OrientationViewType cellOrientations,
                      const BasisType* cellBasis,
                      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);


  template<typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHCurlEvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      typename BasisType::ScalarViewType curlEvalPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<typename basisCoeffsValueType, class ...basisCoeffsProperties,
  typename funValsValueType, class ...funValsProperties,
  typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHCurlBasisCoeffs(Kokkos::DynRankView<basisCoeffsValueType,basisCoeffsProperties...> basisCoeffs,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetAtEvalPoints,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetCurlAtCurlEvalPoints,
      const typename BasisType::ScalarViewType evaluationPoints,
      const typename BasisType::ScalarViewType curlEvalPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);


  template<typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHDivEvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      typename BasisType::ScalarViewType divEvalPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<typename basisCoeffsValueType, class ...basisCoeffsProperties,
  typename funValsValueType, class ...funValsProperties,
  typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHDivBasisCoeffs(Kokkos::DynRankView<basisCoeffsValueType,basisCoeffsProperties...> basisCoeffs,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetAtEvalPoints,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetDivAtDivEvalPoints,
      const typename BasisType::ScalarViewType evaluationPoints,
      const typename BasisType::ScalarViewType divEvalPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);

  template<typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHVolEvaluationPoints(typename BasisType::ScalarViewType evaluationPoints,
      const Kokkos::DynRankView<ortValueType, ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct,
      const EvalPointsType evalPointType = EvalPointsType::TARGET
  );

  template<typename basisCoeffsValueType, class ...basisCoeffsProperties,
  typename funValsValueType, class ...funValsProperties,
  typename BasisType,
  typename ortValueType,       class ...ortProperties>
  static void
  getHVolBasisCoeffs(Kokkos::DynRankView<basisCoeffsValueType,basisCoeffsProperties...> basisCoeffs,
      const Kokkos::DynRankView<funValsValueType,funValsProperties...> targetAtEvalPoints,
      const typename BasisType::ScalarViewType evaluationPoints,
      const Kokkos::DynRankView<ortValueType,   ortProperties...>  cellOrientations,
      const BasisType* cellBasis,
      ProjectionStruct<DeviceType, typename BasisType::scalarType> * projStruct);


  struct ElemSystem {


    std::string systemName_;
    bool matrixIndependentOfCell_;

    ElemSystem (std::string systemName, bool matrixIndependentOfCell) :
      systemName_(systemName), matrixIndependentOfCell_(matrixIndependentOfCell){};



    template<typename ViewType1, typename ViewType2, typename ViewType3, typename ViewType4>
    void solve(ViewType1 basisCoeffs, ViewType2 elemMat, ViewType2 elemRhs, ViewType2 tau,
        ViewType3 w,const  ViewType4 elemDof, ordinal_type n, ordinal_type m=0) {
#ifdef HAVE_INTREPID2_KOKKOSKERNELS
      solveDevice(basisCoeffs, elemMat, elemRhs, tau,
                  w, elemDof, n, m);
#else
      solveHost(basisCoeffs, elemMat, elemRhs, tau,
          w, elemDof, n, m);
#endif

    }

#ifdef HAVE_INTREPID2_KOKKOSKERNELS
    template<typename ViewType1, typename ViewType2, typename ViewType3, typename ViewType4>
    void solveDevice(ViewType1 basisCoeffs, ViewType2 elemMat, ViewType2 elemRhs, ViewType2 taul,
        ViewType3 work,const  ViewType4 elemDof, ordinal_type n, ordinal_type m) {
      using HostDeviceType = Kokkos::Device<Kokkos::DefaultHostExecutionSpace,Kokkos::HostSpace>;

      ordinal_type numCells = basisCoeffs.extent(0);

      if(matrixIndependentOfCell_) {
        auto A0 = Kokkos::subview(elemMat, 0, Kokkos::ALL(), Kokkos::ALL());
        auto tau0 = Kokkos::subview(taul, 0, Kokkos::ALL());

        Kokkos::DynRankView<typename ViewType2::value_type, HostDeviceType> A0_host("A0_host", A0.extent(0),A0.extent(1));
        auto A0_device = Kokkos::create_mirror_view(typename DeviceType::memory_space(), A0_host);
        Kokkos::deep_copy(A0_device, A0);
        Kokkos::deep_copy(A0_host, A0_device);

        for(ordinal_type i=n; i<n+m; ++i)
          for(ordinal_type j=0; j<n; ++j)
            A0_host(i,j) = A0_host(j,i);

        Kokkos::DynRankView<typename ViewType2::value_type, HostDeviceType> tau0_host("A0_host", tau0.extent(0));
        auto tau0_device = Kokkos::create_mirror_view(typename DeviceType::memory_space(), tau0_host);
        auto w0_host = Kokkos::create_mirror_view(Kokkos::subview(work, 0, Kokkos::ALL()));

        //computing QR of A0. QR is saved in A0 and tau0
        KokkosBatched::SerialQR_Internal::invoke(A0_host.extent(0), A0_host.extent(1),
            A0_host.data(), A0_host.stride_0(), A0_host.stride_1(),
            tau0_host.data(), tau0_host.stride_0(), w0_host.data());

        Kokkos::deep_copy(A0_device, A0_host);
        Kokkos::deep_copy(A0, A0_device);
        Kokkos::deep_copy(tau0_device, tau0_host);
        Kokkos::deep_copy(tau0, tau0_device);

        Kokkos::parallel_for (systemName_,
            Kokkos::RangePolicy<ExecSpaceType, int> (0, numCells),
            KOKKOS_LAMBDA (const size_t ic) {
          auto w = Kokkos::subview(work, ic, Kokkos::ALL());

          auto b = Kokkos::subview(elemRhs, ic, Kokkos::ALL());

          //b'*Q0 -> b
          KokkosBatched::SerialApplyQ_RightForwardInternal::invoke(
              1, A0.extent(0), A0.extent(1),
              A0.data(),  A0.stride_0(), A0.stride_1(),
              tau0.data(), tau0.stride_0(),
              b.data(),  1, b.stride_0(),
              w.data());

          // R0^{-1} b -> b
          KokkosBatched::SerialTrsvInternalUpper<KokkosBatched::Algo::Trsv::Unblocked>::invoke(false,
              A0.extent(0),
              1.0,
              A0.data(), A0.stride_0(), A0.stride_1(),
              b.data(),  b.stride_0());

          //scattering b into the basis coefficients
          for(ordinal_type i=0; i<n; ++i){
            basisCoeffs(ic,elemDof(i)) = b(i);
          }
        });

      } else {

        Kokkos::parallel_for (systemName_,
            Kokkos::RangePolicy<ExecSpaceType, int> (0, numCells),
            KOKKOS_LAMBDA (const size_t ic) {

          auto A = Kokkos::subview(elemMat, ic, Kokkos::ALL(), Kokkos::ALL());
          auto tau = Kokkos::subview(taul, ic, Kokkos::ALL());
          auto w = Kokkos::subview(work, ic, Kokkos::ALL());

          for(ordinal_type i=n; i<n+m; ++i)
            for(ordinal_type j=0; j<n; ++j)
              A(i,j) = A(j,i);

          //computing QR of A. QR is saved in A and tau
          KokkosBatched::SerialQR_Internal::invoke(A.extent(0), A.extent(1),
              A.data(), A.stride_0(), A.stride_1(), tau.data(), tau.stride_0(), w.data());

          auto b = Kokkos::subview(elemRhs, ic, Kokkos::ALL());

          //b'*Q -> b
          KokkosBatched::SerialApplyQ_RightForwardInternal::invoke(
              1, A.extent(0), A.extent(1),
              A.data(),  A.stride_0(), A.stride_1(),
              tau.data(), tau.stride_0(),
              b.data(),  1, b.stride_0(),
              w.data());

          // R^{-1} b -> b
          KokkosBatched::SerialTrsvInternalUpper<KokkosBatched::Algo::Trsv::Unblocked>::invoke(false,
              A.extent(0),
              1.0,
              A.data(), A.stride_0(), A.stride_1(),
              b.data(),  b.stride_0());

          //scattering b into the basis coefficients
          for(ordinal_type i=0; i<n; ++i){
            basisCoeffs(ic,elemDof(i)) = b(i);
          }
        });
      }
    }
#endif

    template<typename ViewType1, typename ViewType2, typename ViewType3, typename ViewType4>
    void solveHost(ViewType1 basisCoeffs, ViewType2 elemMat, ViewType2 elemRhs, ViewType2 ,
                   ViewType3, const  ViewType4 elemDof, ordinal_type n, ordinal_type m) {
      using value_type = typename ViewType2::value_type;
      using device_type = DeviceType;
      using host_exec_space = Kokkos::DefaultHostExecutionSpace;
      using host_memory_space = Kokkos::HostSpace;
      using host_device_type = Kokkos::Device<host_exec_space,host_memory_space>;
      using vector_host_type = Kokkos::View<value_type*,host_device_type>;
      using scratch_host_type = Kokkos::View<value_type*,host_exec_space::scratch_memory_space>;
      using matrix_host_type = Kokkos::View<value_type**,Kokkos::LayoutLeft,host_device_type>;
      using do_not_init_tag = Kokkos::ViewAllocateWithoutInitializing;
      using host_team_policy_type = Kokkos::TeamPolicy<host_exec_space>;
      using host_range_policy_type = Kokkos::RangePolicy<host_exec_space>;

      Kokkos::fence();

      const ordinal_type numCells = basisCoeffs.extent(0);
      const ordinal_type numRows = m+n, numCols = n;

      Teuchos::LAPACK<ordinal_type,value_type> lapack;

      Kokkos::View<ordinal_type*,host_device_type> elemDof_host(do_not_init_tag("elemDof_host"), elemDof.extent(0));
      {
        auto elemDof_device = Kokkos::create_mirror_view(typename device_type::memory_space(), elemDof_host);
        Kokkos::deep_copy(elemDof_device, elemDof); Kokkos::fence();
        Kokkos::deep_copy(elemDof_host, elemDof_device);
      }

      auto elemRhs_host = Kokkos::create_mirror_view_and_copy(host_memory_space(), elemRhs);
      auto elemMat_host = Kokkos::create_mirror_view_and_copy(host_memory_space(), elemMat);

      auto basisCoeffs_host = Kokkos::create_mirror_view_and_copy(host_memory_space(), basisCoeffs);

      if (matrixIndependentOfCell_) {
        matrix_host_type A(do_not_init_tag("A"), numRows, numRows);
        {
          for (ordinal_type j=0;j<numRows;++j)
            for (ordinal_type i=0;i<numRows;++i)
              A(i, j) = elemMat_host(0, i, j);

          for (ordinal_type j=0;j<numCols;++j)
            for (ordinal_type i=numCols;i<numRows;++i)
              A(i, j) = A(j, i);
        }
        
        ordinal_type lwork(-1);
        { 
          ordinal_type info(0);
          value_type work[2];
          lapack.GELS('N', 
                      numRows, numRows, numCells,
                      nullptr, numRows,
                      nullptr, numRows,
                      &work[0], lwork,
                      &info);
          lwork = work[0];
        }
        
        matrix_host_type C(do_not_init_tag("C"), numRows, numCells);

        host_range_policy_type policy(0, numCells);
        {
          Kokkos::parallel_for
            ("ProjectionTools::solveHost::matrixIndependentOfCell::pack",
             policy, [=](const ordinal_type & ic) {
              for (ordinal_type i=0;i<numRows;++i)
                C(i,ic) = elemRhs_host(ic, i);
            });
        }
        {
          vector_host_type work(do_not_init_tag("work"), lwork);
          ordinal_type info(0);
          lapack.GELS('N', 
                      numRows, numRows, numCells,
                      A.data(), numRows,
                      C.data(), numRows,
                      work.data(), lwork,
                      &info);
          INTREPID2_TEST_FOR_EXCEPTION
            (info != 0, std::runtime_error, "GELS return non-zero info code");          
        }
        {
          Kokkos::parallel_for
            ("ProjectionTools::solveHost::matrixIndependentOfCell::unpack",
             policy, [=](const ordinal_type & ic) {
              for (ordinal_type i=0;i<numCols;++i)
                basisCoeffs_host(ic,elemDof_host(i)) = C(i,ic);
            });
        }
      } else {
        const ordinal_type level(0);
        host_team_policy_type policy(numCells, 1, 1);

        ordinal_type lwork(-1);
        {
          ordinal_type info(0);
          value_type work[2];
          lapack.GELS('N', 
                      numRows, numRows, 1,
                      nullptr, numRows,
                      nullptr, numRows,
                      &work[0], lwork,
                      &info);
          lwork = work[0];
        }

        const ordinal_type per_team_extent = numRows*numRows + numRows + lwork;
        const ordinal_type per_team_scratch = scratch_host_type::shmem_size(per_team_extent);
        policy.set_scratch_size(level, Kokkos::PerTeam(per_team_scratch));

        Kokkos::parallel_for
          ("ProjectionTools::solveHost::matrixDependentOfCell",
           policy, [=](const typename host_team_policy_type::member_type& member) {
            const ordinal_type ic = member.league_rank();
            
            scratch_host_type scratch(member.team_scratch(level), per_team_extent);
            value_type * sptr = scratch.data();
            
            matrix_host_type A(sptr, numRows, numRows); sptr += A.span();
            for (ordinal_type j=0;j<numRows;++j)
              for (ordinal_type i=0;i<numRows;++i)
                A(i, j) = elemMat_host(ic, i, j);

            for (ordinal_type j=0;j<numCols;++j)
              for (ordinal_type i=numCols;i<numRows;++i)
                A(i, j) = A(j, i);

            vector_host_type c(sptr, numRows); sptr += c.span();
            for (ordinal_type i=0;i<numRows;++i)
              c(i) = elemRhs_host(ic, i);

            vector_host_type work(sptr, lwork); sptr += work.span();
            ordinal_type info(0);
            lapack.GELS('N', 
                        numRows, numRows, 1,
                        A.data(), numRows,
                        c.data(), numRows,
                        work.data(), lwork,
                        &info);
            INTREPID2_TEST_FOR_EXCEPTION
              (info != 0, std::runtime_error, "GELS return non-zero info code");          

            for (ordinal_type i=0;i<numCols;++i) 
              basisCoeffs_host(ic,elemDof_host(i)) = c(i);
          });
      }      
      Kokkos::deep_copy(basisCoeffs, basisCoeffs_host);
    }
  };
  
};

} //Experimental
} //Intrepid2


// include templated function definitions
#include "Intrepid2_ProjectionToolsDefL2.hpp"
#include "Intrepid2_ProjectionToolsDefHGRAD.hpp"
#include "Intrepid2_ProjectionToolsDefHCURL.hpp"
#include "Intrepid2_ProjectionToolsDefHDIV.hpp"
#include "Intrepid2_ProjectionToolsDefHVOL.hpp"

#endif